Disk or Folder Integrity Manager
Posted: Tue May 09, 2017 6:43 pm
This is a tool I wrote a while back to keep track of file changes and detect corruption. It basically makes a CSV for that folder with enough details to detect change.
Where to use it: flash drives, magnetic drives, PCIe drives, RAID 0-6 clusters(I personally use hardware based RAID 0 for gaming, and hardware based RAID 1 or 6 for servers)
What does it run on: Everything PureBASIC does(OSX not tested but should work). I intentionally don't use API hooks etc..
What it's not: Some volume-sandbox like DeepFreeze or S.M.A.R.T monitor tool like SpinRite
Best use cases:
Potential improvements: Cache(although pagefile or swap should take care of bigger scans), CLI interface for batch automated daemons, adaptive Delay() based on CPU %(requires API polling), privilege adoption to handle custom and protected ACLs like SYSTEM and chmod 600 etc..
It averages around 1.8% CPU and 3.4MB RAM on my Windows 10 Arrandale dual-core Laptop doing a scan of my 1TB USB backup drive that has a lot of GiB ISO files.
HOW TO USE IT:
Where to use it: flash drives, magnetic drives, PCIe drives, RAID 0-6 clusters(I personally use hardware based RAID 0 for gaming, and hardware based RAID 1 or 6 for servers)
What does it run on: Everything PureBASIC does(OSX not tested but should work). I intentionally don't use API hooks etc..
What it's not: Some volume-sandbox like DeepFreeze or S.M.A.R.T monitor tool like SpinRite
Best use cases:
- "offline" volume or backup scans where rootkits are expected to be hiding files since antiviruses are purely at the mercy of signature databases even in their heuristic modes. The only malware this won't detect are rare cases like TDL4 that makes it's own custom file system outside of the active volume. Most antivirus's still don't detect that if it's obfuscated during install..
- Detect corrupted or changed file or folder on demand
Potential improvements: Cache(although pagefile or swap should take care of bigger scans), CLI interface for batch automated daemons, adaptive Delay() based on CPU %(requires API polling), privilege adoption to handle custom and protected ACLs like SYSTEM and chmod 600 etc..
It averages around 1.8% CPU and 3.4MB RAM on my Windows 10 Arrandale dual-core Laptop doing a scan of my 1TB USB backup drive that has a lot of GiB ISO files.
HOW TO USE IT:
- Click "Browse" and select a folder or drive
- If you are verifying a folder or drive that already has a valid hashes.txt in it then check the "Verify" box
- Click "Start"
- Wait for "Stop" on "Start" button to turn to "Start"(There is also the progress bar as an indicator). You can click stop at any time.
- Optionally copy and paste the output to "hashes.txt" in the target folder so you can use the results to verify or save time on the next scan.
Code: Select all
EnableExplicit
UseCRC32Fingerprint()
Global counter.l;used for progress bar and file counting
Global base$ ;used for easy relative paths
Global tstate.b ;tells threads to stop
Global Dim scanned$(0) ;filled with what's in existing hashes.txt
Global duration.l ;used for telling time it took
Define event.l ;used for waitwindowevent()
Define etype.l ;used for eventtype()
Define gadget.l ;used for eventgadget()
Declare counterz(path$)
Declare crawl(path$)
Declare Verify()
Declare captain(*val)
;basic GUI creation and loop
If OpenWindow(0,0,0,770,440,"Backup Tool",#PB_Window_ScreenCentered|#PB_Window_MinimizeGadget)
StringGadget(0,0,0,620,25,"")
DisableGadget(0,1)
ButtonGadget(1,620,0,150,25,"Browse")
EditorGadget(2,0,62,383,378)
SetGadgetColor(2,#PB_Gadget_BackColor,RGB(0,0,0))
SetGadgetColor(2,#PB_Gadget_FrontColor,RGB(180,255,0))
ButtonGadget(3,620,28,150,25,"Start")
ProgressBarGadget(4,0,27,450,25,0,100)
CheckBoxGadget(6,506,30,55,18,"Verify")
EditorGadget(7,387,62,383,378)
SetGadgetColor(7,#PB_Gadget_BackColor,RGB(0,0,0))
SetGadgetColor(7,#PB_Gadget_FrontColor,RGB(255,161,0))
Repeat
event=WaitWindowEvent()
etype=EventType()
gadget=EventGadget()
If event=#PB_Event_Gadget
Select gadget
Case 1
If etype=#PB_EventType_LeftClick
SetGadgetText(0,PathRequester("Directory Pick",""))
EndIf
Case 3
If etype=#PB_EventType_LeftClick
If GetGadgetText(3)="Start" And Len(GetGadgetText(0))>0
tstate=#True
CreateThread(@captain(),1)
Else
tstate=#False
EndIf
EndIf
EndSelect
EndIf
Until event=#PB_Event_CloseWindow
EndIf
End
;central thread caller
Procedure captain(*val)
Protected file$
Protected i.l
;disable browse
DisableGadget(1,1)
SetGadgetText(3,"Stop")
SetGadgetText(2,"Counting Files...")
counter=0
counterz(GetGadgetText(0))
;reset output
SetGadgetText(2,"")
SetGadgetText(7,"")
;reset progress bar
SetGadgetAttribute(4,#PB_ProgressBar_Maximum,counter)
SetGadgetAttribute(4,#PB_ProgressBar_Minimum,1)
;reset counter
counter=0
;set timer
duration=Date()
;set base path so output can be made relative
base$=GetGadgetText(0)
;load any old scan
ReDim scanned$(0) : scanned$(0)=""
If FileSize(GetGadgetText(0)+"hashes.txt")>0
If ReadFile(0,GetGadgetText(0)+"hashes.txt",#PB_File_SharedRead|#PB_File_SharedWrite)
While Eof(0)=0
scanned$(ArraySize(scanned$()))=Trim(ReadString(0,#PB_UTF8))
ReDim scanned$(ArraySize(scanned$())+1)
Wend
ReDim scanned$(ArraySize(scanned$())-1)
CloseFile(0)
Else
MessageRequester("Error","hashes.txt found but can't be read")
EndIf
EndIf
If GetGadgetState(6)=#PB_Checkbox_Checked
;verify mode
SetGadgetText(2,"")
If FileSize(GetGadgetText(0)+"hashes.txt")>0 And Len(scanned$(0))>0
AddGadgetItem(2,-1,FormatDate("%dd/%mm/%yy %hh:%ii:%ss",Date()))
verify()
If (ArraySize(scanned$())+1)<GetGadgetAttribute(4,#PB_ProgressBar_Maximum) : AddGadgetItem(2,-1,Str(GetGadgetAttribute(4,#PB_ProgressBar_Maximum)-(ArraySize(scanned$())+1))+" new files since last scan") : EndIf
MessageRequester("","Verify finished in "+FormatDate("%hh:%ii:%ss",Date()-duration))
Else
AddGadgetItem(2,-1,"No hashes found")
EndIf
Else
;put missing old files in diff window on scan mode
If Len(scanned$(0))>0
For i=0 To ArraySize(scanned$())
If FileSize(base$+Mid(scanned$(i),2,FindString(scanned$(i),Chr(34),2)-2))=-1
AddGadgetItem(7,-1,base$+Mid(scanned$(i),2,FindString(scanned$(i),Chr(34),2)-2)+" moved or deleted")
EndIf
Next
EndIf
;build mode
crawl(GetGadgetText(0))
MessageRequester("","Scan finish in "+FormatDate("%hh:%ii:%ss",Date()-duration))
EndIf
;reset progress bar
SetGadgetState(4,1)
;enable browse
DisableGadget(1,0)
SetGadgetText(3,"Start")
counter=1
EndProcedure
;gets file count for progress bar
Procedure counterz(path$)
Protected adir.l
adir=ExamineDirectory(#PB_Any,path$,"*.*")
If adir
While NextDirectoryEntry(adir)
If tstate=#False : Break : EndIf
If DirectoryEntryType(adir)=#PB_DirectoryEntry_Directory
If DirectoryEntryName(adir)<>"." And DirectoryEntryName(adir)<>".." And DirectoryEntryName(adir)<>"$RECYCLE.BIN"
CompilerIf #PB_Compiler_OS = #PB_OS_Linux
counterz(path$+DirectoryEntryName(adir)+"/")
CompilerElseIf #PB_Compiler_OS = #PB_OS_Windows
counterz(path$+DirectoryEntryName(adir)+"\")
CompilerElse
counterz(path$+DirectoryEntryName(adir)+"/")
CompilerEndIf
EndIf
Else
counter=counter+1
EndIf
Wend
FinishDirectory(adir)
EndIf
EndProcedure
;called from crawl. if hashes.txt existed for path we use old hash if file size unchanged
Procedure.b backscan(path$)
Protected i.l
For i = 0 To ArraySize(scanned$())
If Mid(scanned$(i),2,FindString(scanned$(i),Chr(34),2)-2)=ReplaceString(path$,base$,"",#PB_String_NoCase,1,1)
If FileSize(path$)<>Val(StringField(scanned$(i),CountString(scanned$(i),","),","))
AddGadgetItem(7,-1,Mid(scanned$(i),2,FindString(scanned$(i),Chr(34),2)-2)+" size changed")
ProcedureReturn #True
Else
AddGadgetItem(2,-1,scanned$(i))
ProcedureReturn #False
EndIf
EndIf
Next
ProcedureReturn #True
EndProcedure
;the actual csv builder
Procedure crawl(path$)
Protected emdir.l
emdir=ExamineDirectory(#PB_Any,path$,"*.*")
If emdir
While NextDirectoryEntry(emdir)
If tstate=#False : Break : EndIf
Delay(500)
If DirectoryEntryType(emdir)=#PB_DirectoryEntry_Directory
If DirectoryEntryName(emdir)<>"." And DirectoryEntryName(emdir)<>".." And DirectoryEntryName(emdir)<>"$RECYCLE.BIN"
CompilerIf #PB_Compiler_OS = #PB_OS_Windows
crawl(path$+DirectoryEntryName(emdir)+"\")
CompilerElse
crawl(path$+DirectoryEntryName(emdir)+"/")
CompilerEndIf
EndIf
Else
If backscan(path$+DirectoryEntryName(emdir))=#True
CompilerIf #PB_Compiler_OS = #PB_OS_Windows
AddGadgetItem(2,-1,Chr(34)+ReplaceString(path$,base$,"",#PB_String_NoCase,1,1)+DirectoryEntryName(emdir)+Chr(34)+","+FileSize(path$+DirectoryEntryName(emdir))+","+FileFingerprint(path$+DirectoryEntryName(emdir),#PB_Cipher_CRC32))
CompilerElse
AddGadgetItem(2,-1,Chr(34)+ReplaceString(path$,base$,"",#PB_String_NoCase,1,1)+DirectoryEntryName(emdir)+Chr(34)+","+FileSize(path$+DirectoryEntryName(emdir))+","+FileFingerprint(path$+DirectoryEntryName(emdir),#PB_Cipher_CRC32))
CompilerEndIf
EndIf
counter=counter+1
SetGadgetState(4,counter)
EndIf
Wend
FinishDirectory(emdir)
EndIf
EndProcedure
;verifies with existing csv when checkbox checked
Procedure Verify()
Protected i.l
For i = 0 To ArraySize(scanned$())
If tstate=#False : Break : EndIf
Delay(500)
If OSVersion()<>#PB_OS_Windows And CountString(scanned$(i),"\")>0 : scanned$(i)=ReplaceString(scanned$(i),"\","/") : EndIf
If FileSize(base$+Mid(scanned$(i),2,FindString(scanned$(i),Chr(34),2)-2))<>Val(StringField(scanned$(i),CountString(scanned$(i),","),","))
AddGadgetItem(2,-1,base$+Mid(scanned$(i),2,FindString(scanned$(i),Chr(34),2)-2)+" SIZE FAIL")
Else
AddGadgetItem(2,-1,base$+Mid(scanned$(i),2,FindString(scanned$(i),Chr(34),2)-2)+" SIZE MATCH")
EndIf
If FileFingerprint(base$+Mid(scanned$(i),2,FindString(scanned$(i),Chr(34),2)-2),#PB_Cipher_CRC32)<>StringField(scanned$(i),CountString(scanned$(i),",")+1,",")
AddGadgetItem(2,-1,base$+Mid(scanned$(i),2,FindString(scanned$(i),Chr(34),2)-2)+" HASH FAIL")
Else
AddGadgetItem(2,-1,base$+Mid(scanned$(i),2,FindString(scanned$(i),Chr(34),2)-2)+" HASH MATCH")
EndIf
counter=counter+1
SetGadgetState(4,counter)
Next
EndProcedure