Rsync fix corrupt downloads HOWTO PDF Print E-mail
User Rating: / 0
PoorBest 
Written by Ozan Uzun   


I have a very old DS5K series storage. One day DS5K felt lonely, wanted to draw attention (a special kind of love!) and two disks failed at the same time.

 

 


Sadly, some iso files in my NFS share became corrupt.  File sizes are similar to original ones,  but checksum differs.

 

Uploading them from home  to LAB would have taken aprox. 32 days:) 



Again, main problem with big file transfers is the possibility of file corruption. Tools like wget,ftp,sftp will resume but not fix corrupted data.

rsync is the ultimate tool for such occasions, it is available for Linux, AIX, Solaris,MacOS.


Lets fix my corrupted iso's.  4 iso files consists around 16 GB data.
Rsync recipe are below;


–checksum (skip based on checksum, not mod-time & size),
–in-place (to transfer the file in place as rsync normally writes a temporary file, then moves)
–no-whole-file (to not copy the whole file, but use deltas instead).




[root@p8kvm images]# rsync -Pa --checksum --inplace --no-whole-file RHEL-*  -e ssh root@xxxxx://mnt/stg-files/Linux/rhel
The authenticity of host 'xxxxx xxxx)' can't be established.
RSA key fingerprint is f7:db:fa:xx:xx:2d:61:75:7d:c1:e6:75:0d:dc:07:58.
Are you sure you want to continue connecting (yes/no)? yes
Warning: Permanently added 'xxxxxxxx' (RSA) to the list of known hosts.
root@xxxxx's password:
sending incremental file list
RHEL-6.6-20140926.0-Server-ppc64-dvd1.iso
  3396677632 100%   37.73MB/s    0:01:25 (xfer#1, to-check=3/4)
RHEL-7.1-20150219.1-Server-ppc64-dvd1.iso
  3267020800 100%   59.13MB/s    0:00:52 (xfer#2, to-check=2/4)
RHEL-7.2-20151030.0-Server-ppc64-dvd1.iso
  3395985408 100%   65.84MB/s    0:00:49 (xfer#3, to-check=1/4)
RHEL-LE-7.1-20150219.1-Server-ppc64le-dvd1.iso
  3368349696 100%   41.20MB/s    0:01:17 (xfer#4, to-check=0/4)



Four iso files,  finished around 10 minutes:)

I love rsync.

 
< Prev   Next >
Login
61 queries executed
1
SET sql_mode = 'MYSQL40'
2 SELECT folder, element, published, params FROM jos_mambots WHERE published >= 1 AND access <= 0 AND folder = 'system' ORDER BY ordering
3 SELECT template FROM jos_templates_menu WHERE client_id = 0 AND ( menuid = 0 OR menuid = 309 ) ORDER BY menuid DESC LIMIT 1
4 DELETE FROM jos_session WHERE ( ( time < '1548315006' ) AND guest = 0 AND gid > 0 ) OR ( ( time < '1548315006' ) AND guest = 1 AND userid = 0 )
5 SELECT COUNT(*) FROM jos_stats_agents WHERE agent = 'Unknown' AND type = 0
6 UPDATE jos_stats_agents SET hits = ( hits + 1 ) WHERE agent = 'Unknown' AND type = 0
7 SELECT COUNT(*) FROM jos_stats_agents WHERE agent = 'Unknown' AND type = 1
8 UPDATE jos_stats_agents SET hits = ( hits + 1 ) WHERE agent = 'Unknown' AND type = 1
9 SELECT COUNT(*) FROM jos_stats_agents WHERE agent = 'com' AND type = 2
10 UPDATE jos_stats_agents SET hits = ( hits + 1 ) WHERE agent = 'com' AND type = 2
11 SELECT * FROM jos_menu WHERE id = 309
12 SELECT a.*, u.name AS author, u.usertype, cc.name AS category, s.name AS section, g.name AS groups, s.published AS sec_pub, cc.published AS cat_pub, s.access AS sec_access, cc.access AS cat_access, s.id AS sec_id, cc.id as cat_id FROM jos_content AS a LEFT JOIN jos_categories AS cc ON cc.id = a.catid LEFT JOIN jos_sections AS s ON s.id = cc.section AND s.scope = 'content' LEFT JOIN jos_users AS u ON u.id = a.created_by LEFT JOIN jos_groups AS g ON a.access = g.id WHERE a.id = 765 AND ( a.state = 1 OR a.state = -1 ) AND ( a.publish_up = '0000-00-00 00:00:00' OR a.publish_up <= '2019-01-24 08:45' ) AND ( a.publish_down = '0000-00-00 00:00:00' OR a.publish_down >= '2019-01-24 08:45' ) AND a.access <= 0
13 SELECT a.id FROM jos_content AS a WHERE a.catid = 19 AND a.state = 1 AND a.access <= 0 AND ( a.state = 1 OR a.state = -1 ) AND ( a.publish_up = '0000-00-00 00:00:00' OR a.publish_up <= '2019-01-24 08:45' ) AND ( a.publish_down = '0000-00-00 00:00:00' OR a.publish_down >= '2019-01-24 08:45' ) ORDER BY a.created DESC
14 SELECT ROUND( v.rating_sum / v.rating_count ) AS rating, v.rating_count FROM jos_content AS a LEFT JOIN jos_content_rating AS v ON a.id = v.content_id WHERE a.id = 765
15 UPDATE jos_content SET hits = ( hits + 1 ) WHERE id = '765'
16 SELECT hits FROM jos_core_log_items WHERE time_stamp = '2019-01-24' AND item_table = '#__content' AND item_id = '765'
17 UPDATE jos_core_log_items SET hits = ( hits + 1 ) WHERE time_stamp = '2019-01-24' AND item_table = '#__content' AND item_id = '765'
18 SELECT folder, element, published, params FROM jos_mambots WHERE access <= 0 AND folder = 'content' ORDER BY ordering
19 SELECT value FROM jos_bookmarks_prefs WHERE userid = '-1' AND category = 'params' AND name = 'urlkey'
20 SELECT value FROM jos_bookmarks_prefs WHERE userid = '-1' AND category = 'params' AND name = 'snapshotactiv'
21 SELECT value FROM jos_bookmarks_prefs WHERE userid = '-1' AND category = 'params' AND name = 'snapshotother'
22 SELECT id FROM jos_mambots WHERE element = 'mosbookmarks' AND folder = 'content'
23 SELECT * FROM jos_mambots WHERE id = '32'
24 SELECT name, value FROM jos_bookmarks_prefs WHERE userid = '-1' AND category = 'params' ORDER BY name
25 SELECT c.id, c.parent FROM jos_bookmarks_categories as c WHERE c.id = -1
26 SELECT * FROM jos_bookmarks_columns WHERE category='' AND custom='0'
27 SELECT * FROM jos_bookmarks_columns WHERE category='' ORDER BY ordering
28 SELECT * FROM jos_bookmarks_columns WHERE category='admin'
29 SELECT * FROM jos_bookmarks_columns WHERE category='admin' ORDER BY ordering
30 SELECT id FROM jos_mambots WHERE element = 'plugin_jw_allvideos' AND folder = 'content'
31 SELECT * FROM jos_mambots WHERE id = '27'
32 SELECT id FROM jos_mambots WHERE element = 'plugin_jw_allvideos' AND folder = 'content'
33 SELECT * FROM jos_mambots WHERE id = '27'
34 SELECT id FROM jos_mambots WHERE element = 'plugin_jw_allvideos' AND folder = 'content'
35 SELECT * FROM jos_mambots WHERE id = '27'
36 SELECT id FROM jos_mambots WHERE element = 'plugin_jw_allvideos' AND folder = 'content'
37 SELECT * FROM jos_mambots WHERE id = '27'
38 SELECT params FROM jos_mambots WHERE element = 'mosemailcloak' AND folder = 'content'
39 SELECT a.* FROM jos_components AS a WHERE ( a.admin_menu_link = 'option=com_syndicate' OR a.admin_menu_link = 'option=com_syndicate&hidemainmenu=1' ) AND a.option = 'com_syndicate'
40 SELECT m.id FROM jos_modules AS m WHERE m.module = 'mod_rssfeed' AND m.published = 1
41 SELECT id, title, module, position, content, showtitle, params FROM jos_modules AS m INNER JOIN jos_modules_menu AS mm ON mm.moduleid = m.id WHERE m.published = 1 AND m.access <= 0 AND m.client_id != 1 AND ( mm.menuid = 0 OR mm.menuid = 309 ) ORDER BY ordering
42 SELECT id FROM jos_menu WHERE link = 'index.php?option=com_search' AND published = 1
43 SELECT m.*, sum(case when p.published=1 then 1 else 0 end) as cnt FROM jos_menu AS m LEFT JOIN jos_menu AS p ON p.parent = m.id WHERE m.menutype='rt_splitSuckerFish' AND m.published='1' AND m.access <= '0' GROUP BY m.id ORDER BY m.parent, m.ordering
44 SELECT m.* FROM jos_menu AS m WHERE menutype='rt_splitSuckerFish' AND m.published='1'
45 SELECT * FROM jos_banner WHERE showBanner=1
46 SELECT id FROM jos_mambots WHERE element = 'mosbookmarks' AND folder = 'content'
47 SELECT * FROM jos_mambots WHERE id = '32'
48 SELECT name, value FROM jos_bookmarks_prefs WHERE userid = '-1' AND category = 'params' ORDER BY name
49 SELECT c.id, c.parent FROM jos_bookmarks_categories as c WHERE c.id = -1
50 SELECT * FROM jos_bookmarks_columns WHERE category='' AND custom='0'
51 SELECT * FROM jos_bookmarks_columns WHERE category='' ORDER BY ordering
52 SELECT * FROM jos_bookmarks_columns WHERE category='admin'
53 SELECT * FROM jos_bookmarks_columns WHERE category='admin' ORDER BY ordering
54 SELECT id FROM jos_mambots WHERE element = 'plugin_jw_allvideos' AND folder = 'content'
55 SELECT * FROM jos_mambots WHERE id = '27'
56 SELECT id FROM jos_mambots WHERE element = 'plugin_jw_allvideos' AND folder = 'content'
57 SELECT * FROM jos_mambots WHERE id = '27'
58 SELECT id FROM jos_mambots WHERE element = 'plugin_jw_allvideos' AND folder = 'content'
59 SELECT * FROM jos_mambots WHERE id = '27'
60 SELECT id FROM jos_mambots WHERE element = 'plugin_jw_allvideos' AND folder = 'content'
61 SELECT * FROM jos_mambots WHERE id = '27'