Compare 2 sets of directories, delete dupes
Posted: Sat May 05, 2007 2:34 pm
Compare the contents of two directories. The master set of directories loads into the top listicongadget and the comparison directories into the bottom list.
When you run the comparison, any duplicate files in the top (master) list that exist in the bottom list are deleted from the bottom (comparison) list and removed and removed off disk as well.
Each time you load files and compare, 3 files are created on disk for reference. MasterList.Txt, CompareList.Txt and DeleteList.Txt
For those of you without the PV designer, comment out any line starting with PV_. Thanks to Rings for letting me use the partial file MD5 code.
When you run the comparison, any duplicate files in the top (master) list that exist in the bottom list are deleted from the bottom (comparison) list and removed and removed off disk as well.
Each time you load files and compare, 3 files are created on disk for reference. MasterList.Txt, CompareList.Txt and DeleteList.Txt
For those of you without the PV designer, comment out any line starting with PV_. Thanks to Rings for letting me use the partial file MD5 code.
Code: Select all
;============================================================================================================================
;
;============================================================================================================================
Enumeration 1
#Window_filecompare
EndEnumeration
#WindowIndex = #PB_Compiler_EnumerationValue
Enumeration 1
#Gadget_filecompare_fmain
#Gadget_filecompare_masterlist
#Gadget_filecompare_comparelist
#Gadget_filecompare_fcontrol
#Gadget_filecompare_bgetdir1
#Gadget_filecompare_bgetdir2
#Gadget_filecompare_brun
#Gadget_filecompare_lmasterlist
#Gadget_filecompare_lcomparelist
EndEnumeration
#GadgetIndex=#PB_Compiler_EnumerationValue
Enumeration 1
#StatusBar_filecompare
EndEnumeration
#StatusBarIndex = #PB_Compiler_EnumerationValue
#StatusBar_filecompare_messages = 0
#StatusBar_filecompare_records = 1
Enumeration 1
#Image_filecompare_bgetdir1
#Image_filecompare_bgetdir2
#Image_filecompare_brun
EndEnumeration
#ImageIndex = #PB_Compiler_EnumerationValue
;============================================================================================================================
;
;============================================================================================================================
CatchImage(#Image_filecompare_bgetdir1, ?_OPT_filecompare_bgetdir1)
CatchImage(#Image_filecompare_bgetdir2, ?_OPT_filecompare_bgetdir1)
CatchImage(#Image_filecompare_brun, ?_OPT_filecompare_brun)
;============================================================================================================================
;
;============================================================================================================================
DataSection
_OPT_filecompare_bgetdir1 : IncludeBinary "Images\open48x48.ico"
_OPT_filecompare_brun : IncludeBinary "Images\run48x48.ico"
EndDataSection
;============================================================================================================================
;
;============================================================================================================================
Procedure.l Window_filecompare()
If OpenWindow(#Window_filecompare,22,65,800,600,"Compare and delete duplicate files between 2 sets of directories",#PB_Window_SystemMenu|#PB_Window_MinimizeGadget|#PB_Window_ScreenCentered|#PB_Window_Invisible)
Brush.LOGBRUSH\lbColor=12632256
SetClassLong_(WindowID(#Window_filecompare),#GCL_HBRBACKGROUND,CreateBrushIndirect_(Brush))
If CreateGadgetList(WindowID(#Window_filecompare))
Frame3DGadget(#Gadget_filecompare_fmain,0,0,800,515,"")
ListIconGadget(#Gadget_filecompare_masterlist,5,10,790,250,"Directory",0,#PB_ListIcon_FullRowSelect|#PB_ListIcon_AlwaysShowSelection)
SendMessage_(GadgetID(#Gadget_filecompare_masterlist),#LVM_SETBKCOLOR,0,12632256)
SendMessage_(GadgetID(#Gadget_filecompare_masterlist),#LVM_SETTEXTBKCOLOR,0,12632256)
AddGadgetColumn(#Gadget_filecompare_masterlist,1,"",444)
AddGadgetColumn(#Gadget_filecompare_masterlist,2,"Size",80)
AddGadgetColumn(#Gadget_filecompare_masterlist,3,"Attrib",80)
AddGadgetColumn(#Gadget_filecompare_masterlist,4,"MD5",160)
SetGadgetFont(#Gadget_filecompare_masterlist,LoadFont(#Gadget_filecompare_masterlist,"Arial",12,0))
ListIconGadget(#Gadget_filecompare_comparelist,5,265,790,245,"Directory",0,#PB_ListIcon_FullRowSelect|#PB_ListIcon_AlwaysShowSelection)
SendMessage_(GadgetID(#Gadget_filecompare_comparelist),#LVM_SETBKCOLOR,0,12632256)
SendMessage_(GadgetID(#Gadget_filecompare_comparelist),#LVM_SETTEXTBKCOLOR,0,12632256)
AddGadgetColumn(#Gadget_filecompare_comparelist,1,"",444)
AddGadgetColumn(#Gadget_filecompare_comparelist,2,"Size",80)
AddGadgetColumn(#Gadget_filecompare_comparelist,3,"Attrib",80)
AddGadgetColumn(#Gadget_filecompare_comparelist,4,"MD5",160)
SetGadgetFont(#Gadget_filecompare_comparelist,LoadFont(#Gadget_filecompare_comparelist,"Arial",12,0))
Frame3DGadget(#Gadget_filecompare_fcontrol,0,515,800,65,"")
ButtonImageGadget(#Gadget_filecompare_bgetdir1,5,525,50,50,ImageID(#Image_filecompare_bgetdir1))
ButtonImageGadget(#Gadget_filecompare_bgetdir2,55,525,50,50,ImageID(#Image_filecompare_bgetdir2))
ButtonImageGadget(#Gadget_filecompare_brun,105,525,50,50,ImageID(#Image_filecompare_brun))
TextGadget(#Gadget_filecompare_lmasterlist,160,525,635,20,"",#PB_Text_Center)
PVDynamic_AddColorGadget(#Gadget_filecompare_lmasterlist,0,-1)
SetGadgetFont(#Gadget_filecompare_lmasterlist,LoadFont(#Gadget_filecompare_lmasterlist,"Arial",12,0))
TextGadget(#Gadget_filecompare_lcomparelist,160,555,635,20,"",#PB_Text_Center)
PVDynamic_AddColorGadget(#Gadget_filecompare_lcomparelist,0,-1)
SetGadgetFont(#Gadget_filecompare_lcomparelist,LoadFont(#Gadget_filecompare_lcomparelist,"Arial",12,0))
CreateStatusBar(#StatusBar_filecompare,WindowID(#Window_filecompare))
AddStatusBarField(670)
AddStatusBarField(130)
HideWindow(#Window_filecompare,0)
ProcedureReturn WindowID(#Window_filecompare)
EndIf
EndIf
EndProcedure
;============================================================================================================================
; declarations
;============================================================================================================================
Declare GetDir1()
Declare GetDir2()
Declare RunCompare()
Declare SearchEngine(SearchDir.s, Gadget.l)
Declare.s MD5PartCheck(Filename.s, Maxsize) ; Build md5 sum for a partial file, (c) 2007 by Siegfried Rings, Windows only
Declare.s GetAttribMask(fattribute.s) ; Get the windows attribute for the specified file with a formatted mask
Declare.s GetFileSize(Filename.s) ; Get the properly formatted file size for a passed filename
;============================================================================================================================
; myconstants
;============================================================================================================================
Structure programdata
quitvalue.l
masterpath.s
masterset.l
masterheader.s
compareset.l
compareheader.s
comparepath.s
EndStructure
;============================================================================================================================
;
;============================================================================================================================
Global program.programdata
;============================================================================================================================
;
;============================================================================================================================
program\masterheader = "FileName (Master list) -- "
program\compareheader = "FileName (Comparison list) -- "
;============================================================================================================================
;
;============================================================================================================================
Enumeration #GadgetIndex
#Shortcut_exit
#Splitter
EndEnumeration
;============================================================================================================================
; Get the list of directories from which you will be comparing
;============================================================================================================================
Procedure GetDir1()
If program\masterset = 1
ClearGadgetItemList(#Gadget_filecompare_masterlist)
program\masterset = 0
program\masterpath = ""
SetGadgetText(#Gadget_filecompare_lmasterlist, "")
EndIf
program\masterpath = PathRequester("Master directory","")
If program\masterpath <> ""
program\masterset = 1
SetGadgetText(#Gadget_filecompare_lmasterlist, "Master list [ " + program\masterpath + " ]")
StatusBarText(#StatusBar_filecompare, #StatusBar_filecompare_messages, "Getting " + program\masterpath + " files, please wait", 0)
SearchEngine(program\masterpath, #Gadget_filecompare_masterlist)
EndIf
EndProcedure
;============================================================================================================================
; Get the secondary list that you will be comparing to and deleting from
;============================================================================================================================
Procedure GetDir2()
If program\compareset = 1
ClearGadgetItemList(#Gadget_filecompare_comparelist)
program\compareset = 0
program\comparepath = ""
SetGadgetText(#Gadget_filecompare_lcomparelist, "")
EndIf
program\comparepath = PathRequester("Comparison directory","")
If program\comparepath <> ""
program\compareset = 1
SetGadgetText(#Gadget_filecompare_lcomparelist, "Comparison list [ " + program\comparepath + " ]")
StatusBarText(#StatusBar_filecompare, #StatusBar_filecompare_messages, "Getting " + program\comparepath + " files, please wait", 0)
SearchEngine(program\comparepath, #Gadget_filecompare_comparelist)
EndIf
EndProcedure
;============================================================================================================================
; Universal, recursive search engine used by many functions
;============================================================================================================================
Procedure SearchEngine(SearchDir.s, Gadget.l)
FileIdm.l = CreateFile(#PB_Any, "MasterList.txt")
FileIdc.l = CreateFile(#PB_Any, "CompareList.txt")
Counter.l = 0
GadgetHeading.s = GetGadgetItemText(Gadget.l, -1, 0)
ClearGadgetItemList(Gadget.l)
NewList FoundDirs.s()
If SearchDir.s <> ""
If Right(SearchDir.s, 1) = "\"
SearchDir.s = Left(SearchDir.s, Len(SearchDir.s) - 1)
EndIf
AddElement(FoundDirs.s())
FoundDirs.s() = SearchDir.s
Index = 0
Repeat
SelectElement(FoundDirs.s(), Index)
If ExamineDirectory(0, FoundDirs.s(), "*.*")
Path.s = FoundDirs.s() + "\"
While NextDirectoryEntry(0)
Filename.s = DirectoryEntryName(0)
Select DirectoryEntryType(0)
Case 1
fFile.s = Path + Filename ; Concatenate the full path
fSize.s = GetFileSize(fFile.s) ; Get the file size
fAttrib.s = GetAttribMask(fFile.s) ; Get file attributes
fMd5.s = MD5PartCheck(fFile.s, 50000) ; Get the MD5 sum
While WindowEvent()
Wend
AddGadgetItem(Gadget.l, -1, Path + Chr(10) + Filename + Chr(10) + fSize + Chr(10) + fAttrib + Chr(10) + fMd5)
Counter + 1
If Gadget.l = #Gadget_filecompare_masterlist ; Update the gadget header
SetGadgetItemText(Gadget.l, -1, program\masterheader + Str(Counter) + " File(s)", 1)
If FileIdm
WriteStringN(FileIdm, Path + "|" + Filename + "|" + fSize + "|" + fAttrib + "|" + fMd5)
EndIf
ElseIf Gadget.l = #Gadget_filecompare_comparelist
SetGadgetItemText(Gadget.l, -1, program\compareheader + Str(Counter) + " File(s)", 1)
If FileIdc
WriteStringN(FileIdc, Path + "|" + Filename + "|" + fSize + "|" + fAttrib + "|" + fMd5)
EndIf
EndIf
SendMessage_(GadgetID(Gadget.l), #LVM_ENSUREVISIBLE, Counter - 1, 0)
Case 2
Filename.s = DirectoryEntryName(0)
If Filename.s <> ".." And Filename.s <> "."
AddElement(FoundDirs())
FoundDirs() = Path + Filename.s
EndIf
EndSelect
Wend
EndIf
Index + 1
Until Index > CountList(FoundDirs()) -1
EndIf
If FileIdm
CloseFile(FileIdm)
ElseIf FileIdc
CloseFile(FileIdc)
EndIf
EndProcedure
;============================================================================================================================
; Compare the two sets of directories and delete matchesfrom the bottom one
;============================================================================================================================
Procedure RunCompare()
FileIdd.l = CreateFile(#PB_Any, "DeletedList.txt") ; Create a deleted files list
NumMaster = CountGadgetItems(#Gadget_filecompare_masterlist) - 1 ; How many files in master list
NumCompare = CountGadgetItems(#Gadget_filecompare_comparelist) - 1 ; How many files in comparison list
PseudoCounter = NumCompare ; Backwards step counter
For MasterFiles = NumMaster To 0 Step -1 ; Iterate through master list
CurrentMasterPath.s = GetGadgetItemText(#Gadget_filecompare_masterlist, MasterFiles, 0) ; Get path
CurrentMasterFile.s = GetGadgetItemText(#Gadget_filecompare_masterlist, MasterFiles, 1) ; Get file
CurrentMasterFull.s = CurrentMasterPath.s + CurrentMasterFile.s ; File and path
CurrentMasterMd5.s = GetGadgetItemText(#Gadget_filecompare_masterlist, MasterFiles, 4) ; Get file
SendMessage_(GadgetID(#Gadget_filecompare_masterlist), #LVM_ENSUREVISIBLE, MasterFiles, 0) ; Scroll line into view
SetGadgetItemText(#Gadget_filecompare_masterlist, -1, "Processing Line Number " + Str(MasterFiles), 1)
For CompareFiles = NumCompare To 0 Step -1 ; Iterate through comparison list
CurrentComparePath.s = GetGadgetItemText(#Gadget_filecompare_comparelist, CompareFiles, 0) ; Get path
CurrentCompareFile.s = GetGadgetItemText(#Gadget_filecompare_comparelist, CompareFiles, 1) ; Get file
CurrentCompareFull.s = CurrentComparePath.s + CurrentCompareFile.s ; File and path
CurrentCompareMd5.s = GetGadgetItemText(#Gadget_filecompare_comparelist, CompareFiles, 4) ; Get file
If CurrentCompareFull.s = CurrentMasterFull.s And CurrentCompareMd5.s = CurrentMasterMd5.s ; Do the two files match
PseudoCounter - 1 ; Backwards counter -1
While WindowEvent() ; Prevent forms from greying out
Wend
DeleteFile(CurrentCompareFull.s) ; Delete matching files
If FileIdd ; If delete list is open, write deleted file to it
WriteStringN(FileIdd, CurrentCompareFull.s) ; Write the string now
EndIf ; End comparison
SendMessage_(GadgetID(#Gadget_filecompare_comparelist), #LVM_DELETEITEM, CompareFiles, 0) ; Delete it now
SetGadgetItemText(#Gadget_filecompare_comparelist, -1, "Processing Line Number " + Str(PseudoCounter), 1)
EndIf
Next CompareFiles
Next MasterFiles
If FileIdd
CloseFile(FileIdd)
EndIf
EndProcedure
;============================================================================================================================
; Build md5 sum for a partial file, (c) 2007 by Siegfried Rings, Windows only
;============================================================================================================================
Procedure.s MD5PartCheck(Filename.s, Maxsize)
length = FileSize(Filename)
If Length < 1
ProcedureReturn ""
EndIf
If Length > Maxsize
Length = Maxsize
EndIf ; Check for max 50000 bytes, should be enough md5.s = ""
handle = CreateFile_(Filename, #GENERIC_READ, #FILE_SHARE_READ, 0, #OPEN_EXISTING, #FILE_ATTRIBUTE_NORMAL, 0)
If handle
MapHandle = CreateFileMapping_(handle, 0, #PAGE_READONLY, 0, length, 0)
If MapHandle
ViewHandle = MapViewOfFile_(MapHandle, #FILE_MAP_READ, 0, 0, 0)
If ViewHandle
md5.s = MD5Fingerprint(ViewHandle, length)
UnmapViewOfFile_(ViewHandle)
EndIf
CloseHandle_(MapHandle) ; okay, now close the Maphandle
EndIf
CloseHandle_(handle) ; okay, now close the Filehandle
ProcedureReturn md5.s
EndIf
EndProcedure
;============================================================================================================================
; Get the windows attribute for the specified file with a formatted mask
;============================================================================================================================
;
Procedure.s GetAttribMask(fattribute.s)
mask.s = "-----"
r = GetFileAttributes_(fattribute.s)
If r & #FILE_ATTRIBUTE_ARCHIVE
mask.s = "A" + Mid(mask.s, 2, 5)
EndIf
If r & #FILE_ATTRIBUTE_COMPRESSED
mask.s = Left(mask.s, 1) + "C" + Mid(mask.s, 3, 3)
EndIf
If r & #FILE_ATTRIBUTE_HIDDEN
mask.s = Left(mask.s, 2) + "H" + Mid(mask.s, 4, 2)
EndIf
If r & #FILE_ATTRIBUTE_READONLY
mask.s = Left(mask.s, 3) + "R" + Mid(mask.s, 5, 1)
EndIf
If r & #FILE_ATTRIBUTE_SYSTEM
mask.s = Left(mask.s, 4) + "S"
EndIf
ProcedureReturn mask.s
EndProcedure
;============================================================================================================================
; Get the properly formatted file size for a passed filename
;============================================================================================================================
Procedure.s GetFileSize(Filename.s)
fSize = FileSize(Filename.s)
If fSize = -1
sSize.s = "Missing!"
ElseIf fSize = -2
sSize.s = "Directory"
ElseIf fSize <= 1023
sSize.s = Str(fSize) + " b"
ElseIf fSize > 1023 And fSize < 1048575
sSize.s = Str(fSize) + " Kb"
ElseIf fSize > 1048576 And fSize < 1073741823
sSize.s = Str(fSize) + " Mb"
ElseIf fSize > 1073741824 And fSize < 1099511627775
sSize.s = Str(fSize) + " Gb"
EndIf
ProcedureReturn sSize.s
EndProcedure
;============================================================================================================================
; main code
;============================================================================================================================
If Window_filecompare()
AddKeyboardShortcut(#Window_filecompare, #PB_Shortcut_Alt | #PB_Shortcut_X, #Shortcut_exit)
SplitterGadget(#Splitter, 4, 9, 791, 502, #Gadget_filecompare_masterlist, #Gadget_filecompare_comparelist, #PB_Splitter_Separator)
program\quitvalue = 0
Repeat
Select WaitWindowEvent()
Case #PB_Event_CloseWindow
If EventWindow() = #Window_filecompare
program\quitvalue = 1
EndIf
Case #PB_Event_Menu
Select EventMenu()
Case #Shortcut_exit : program\quitvalue = 1
EndSelect
Case #PB_Event_Gadget
Select EventGadget()
Case #Gadget_filecompare_bgetdir1 : GetDir1()
Case #Gadget_filecompare_bgetdir2 : GetDir2()
Case #Gadget_filecompare_brun : RunCompare()
EndSelect
EndSelect
Until program\quitvalue
CloseWindow(#Window_filecompare)
EndIf
End