This is what I used to aggregate my tables, cherry picking only the fields that I was interested in. The ACMA database went from 1.7GB to 52MB.
Code: Select all
; // Check if doofus has turned thread safety on //
CompilerIf Not #PB_Compiler_Thread ; Check to see if thread safe environment is running
CompilerError "Enable ThreadSafe!" ; Warn the user that thread safe is not enabled
CompilerEndIf ; End of this conditional test
UseSQLiteDatabase() ;
Structure composite_data
; licence
LICENCE_TYPE_NAME.s ; licence
LICENCE_CATEGORY_NAME.s ; licence
DATE_ISSUED.s ; licence
DATE_OF_EFFECT.s ; licence
DATE_OF_EXPIRY.s ; licence
STATUS.s ; licence
STATUS_TEXT.s ; licence
; client
LICENCEE.s ; client, Linked to licence CLIENT_NO
TRADING_NAME.s ; client
ACN.s ; client
ABN.s ; client
POSTAL_STREET.s ; client
POSTAL_SUBURB.s ; client
POSTAL_STATE.s ; client
POSTAL_POSTCODE.s ; client
; device_details
CALL_SIGN.s
EndStructure
Structure Programdata ;
CurrentDirectory.s ;
DatabaseName.s ;
DatabaseHandle.i ;
DatabaseExists.i ;
QueryString.s ;
UpdateString.s ;
EndStructure ;
Global Program.Programdata ;
Program\CurrentDirectory.s = GetCurrentDirectory() ;
Program\DatabaseName.s = "E:\ACMA amateur radio database.sqlite" ;
Program\DatabaseHandle.i = #False ;
Program\QueryString.s = #Empty$ ;
Program\UpdateString.s = #Empty$ ;
Program\DatabaseExists.i = #False ;
Global NewList Slimtable.composite_data() ;
Program\DatabaseHandle.i = OpenDatabase(#PB_Any, Program\DatabaseName.s, #Empty$, #Empty$, #PB_Database_SQLite)
If Program\DatabaseHandle.i <> #False
CurrentQuery.s = "SELECT " +
"m.LICENCE_TYPE_NAME, " +
"m.LICENCE_CATEGORY_NAME, " +
"m.DATE_ISSUED, " +
"m.DATE_OF_EFFECT, " +
"m.DATE_OF_EXPIRY, " +
"m.STATUS, " +
"m.STATUS_TEXT, " +
"n.LICENCEE, " +
"n.TRADING_NAME, " +
"n.ACN, " +
"n.ABN, " +
"n.POSTAL_STREET, " +
"n.POSTAL_SUBURB, " +
"n.POSTAL_STATE, " +
"n.POSTAL_POSTCODE, " +
"c.CALL_SIGN " +
"FROM licence AS m " +
"LEFT JOIN client AS n " +
"ON m.CLIENT_NO = n.CLIENT_NO " +
"LEFT JOIN device_details AS c " +
"ON m.LICENCE_NO = c.LICENCE_NO " +
"WHERE c.CALL_SIGN NOT NULL"
If DatabaseQuery(Program\DatabaseHandle.i, CurrentQuery.s) <> #False
While NextDatabaseRow(Program\DatabaseHandle.i)
AddElement(Slimtable())
Slimtable()\LICENCE_TYPE_NAME.s = GetDatabaseString(Program\DatabaseHandle.i, 0)
Slimtable()\LICENCE_CATEGORY_NAME.s = GetDatabaseString(Program\DatabaseHandle.i, 1)
Slimtable()\DATE_ISSUED.s = GetDatabaseString(Program\DatabaseHandle.i, 2)
Slimtable()\DATE_OF_EFFECT.s = GetDatabaseString(Program\DatabaseHandle.i, 3)
Slimtable()\DATE_OF_EXPIRY.s = GetDatabaseString(Program\DatabaseHandle.i, 4)
Slimtable()\STATUS.s = GetDatabaseString(Program\DatabaseHandle.i, 5)
Slimtable()\STATUS_TEXT.s = GetDatabaseString(Program\DatabaseHandle.i, 6)
Slimtable()\LICENCEE.s = GetDatabaseString(Program\DatabaseHandle.i, 7)
Slimtable()\TRADING_NAME.s = GetDatabaseString(Program\DatabaseHandle.i, 8)
Slimtable()\ACN.s = GetDatabaseString(Program\DatabaseHandle.i, 9)
Slimtable()\ABN.s = GetDatabaseString(Program\DatabaseHandle.i, 10)
Slimtable()\POSTAL_STREET.s = GetDatabaseString(Program\DatabaseHandle.i, 11)
Slimtable()\POSTAL_SUBURB.s = GetDatabaseString(Program\DatabaseHandle.i, 12)
Slimtable()\POSTAL_STATE.s = GetDatabaseString(Program\DatabaseHandle.i, 13)
Slimtable()\POSTAL_POSTCODE.s = GetDatabaseString(Program\DatabaseHandle.i, 14)
Slimtable()\CALL_SIGN.s = GetDatabaseString(Program\DatabaseHandle.i, 15)
Wend
FinishDatabaseQuery(Program\DatabaseHandle.i)
Else
Debug "The database query failed or contained no results:: " + DatabaseError()
EndIf
If ListSize(Slimtable())
CurrentUpdate.s = "CREATE TABLE composite(" +
"LICENCE_TYPE_NAME TEXT, " +
"LICENCE_CATEGORY_NAME TEXT, " +
"DATE_ISSUED TEXT, " +
"DATE_OF_EFFECT TEXT, " +
"DATE_OF_EXPIRY TEXT, " +
"STATUS TEXT, " +
"STATUS_TEXT TEXT, " +
"LICENCEE TEXT, " +
"TRADING_NAME TEXT, " +
"ACN TEXT, " +
"ABN TEXT, " +
"POSTAL_STREET TEXT, " +
"POSTAL_SUBURB TEXT, " +
"POSTAL_STATE TEXT, " +
"POSTAL_POSTCODE TEXT, " +
"CALL_SIGN TEXT, " +
"Recordid INTEGER PRIMARY KEY AUTOINCREMENT)"
If DatabaseUpdate(Program\DatabaseHandle.i, CurrentUpdate.s) <> #False
DatabaseUpdate(Program\DatabaseHandle.i, "BEGIN TRANSACTION")
ForEach Slimtable()
SetDatabaseString(Program\DatabaseHandle.i, 0, Slimtable()\LICENCE_TYPE_NAME.s)
SetDatabaseString(Program\DatabaseHandle.i, 1, Slimtable()\LICENCE_CATEGORY_NAME.s)
SetDatabaseString(Program\DatabaseHandle.i, 2, Slimtable()\DATE_ISSUED.s)
SetDatabaseString(Program\DatabaseHandle.i, 3, Slimtable()\DATE_OF_EFFECT.s)
SetDatabaseString(Program\DatabaseHandle.i, 4, Slimtable()\DATE_OF_EXPIRY.s)
SetDatabaseString(Program\DatabaseHandle.i, 5, Slimtable()\STATUS.s)
SetDatabaseString(Program\DatabaseHandle.i, 6, Slimtable()\STATUS_TEXT.s)
SetDatabaseString(Program\DatabaseHandle.i, 7, Slimtable()\LICENCEE.s)
SetDatabaseString(Program\DatabaseHandle.i, 8, Slimtable()\TRADING_NAME.s)
SetDatabaseString(Program\DatabaseHandle.i, 9, Slimtable()\ACN.s)
SetDatabaseString(Program\DatabaseHandle.i, 10, Slimtable()\ABN.s)
SetDatabaseString(Program\DatabaseHandle.i, 11, Slimtable()\POSTAL_STREET.s)
SetDatabaseString(Program\DatabaseHandle.i, 12, Slimtable()\POSTAL_SUBURB.s)
SetDatabaseString(Program\DatabaseHandle.i, 13, Slimtable()\POSTAL_STATE.s)
SetDatabaseString(Program\DatabaseHandle.i, 14, Slimtable()\POSTAL_POSTCODE.s)
SetDatabaseString(Program\DatabaseHandle.i, 15, Slimtable()\CALL_SIGN.s)
CurrentInsert.s = "INSERT INTO composite(" +
"LICENCE_TYPE_NAME, " +
"LICENCE_CATEGORY_NAME, " +
"DATE_ISSUED, " +
"DATE_OF_EFFECT, " +
"DATE_OF_EXPIRY, " +
"STATUS, " +
"STATUS_TEXT, " +
"LICENCEE, " +
"TRADING_NAME, " +
"ACN, " +
"ABN, " +
"POSTAL_STREET, " +
"POSTAL_SUBURB, " +
"POSTAL_STATE, " +
"POSTAL_POSTCODE, " +
"CALL_SIGN) " +
"VALUES(?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)"
If DatabaseUpdate(Program\DatabaseHandle.i, CurrentInsert.s) <> #False
Debug "Record added to table composite:: " + Slimtable()\CALL_SIGN.s
Else
Debug "The new record failed to be added to the composite table:: " + DatabaseError()
EndIf
Next Slimtable()
DatabaseUpdate(Program\DatabaseHandle.i, "COMMIT")
; Let's drop the un-needed tables now
DatabaseUpdate(Program\DatabaseHandle.i, "DROP TABLE access_area")
DatabaseUpdate(Program\DatabaseHandle.i, "DROP TABLE access_area_raw")
DatabaseUpdate(Program\DatabaseHandle.i, "DROP TABLE antenna")
DatabaseUpdate(Program\DatabaseHandle.i, "DROP TABLE antenna_pattern")
DatabaseUpdate(Program\DatabaseHandle.i, "DROP TABLE antenna_pattern_raw")
DatabaseUpdate(Program\DatabaseHandle.i, "DROP TABLE antenna_polarity")
DatabaseUpdate(Program\DatabaseHandle.i, "DROP TABLE antenna_polarity_raw")
DatabaseUpdate(Program\DatabaseHandle.i, "DROP TABLE antenna_raw")
DatabaseUpdate(Program\DatabaseHandle.i, "DROP TABLE applic_text_block")
DatabaseUpdate(Program\DatabaseHandle.i, "DROP TABLE applic_text_block_raw")
DatabaseUpdate(Program\DatabaseHandle.i, "DROP TABLE auth_spectrum_area")
DatabaseUpdate(Program\DatabaseHandle.i, "DROP TABLE auth_spectrum_area_raw")
DatabaseUpdate(Program\DatabaseHandle.i, "DROP TABLE auth_spectrum_freq")
DatabaseUpdate(Program\DatabaseHandle.i, "DROP TABLE auth_spectrum_freq_raw")
DatabaseUpdate(Program\DatabaseHandle.i, "DROP TABLE bsl")
DatabaseUpdate(Program\DatabaseHandle.i, "DROP TABLE bsl_area")
DatabaseUpdate(Program\DatabaseHandle.i, "DROP TABLE bsl_area_raw")
DatabaseUpdate(Program\DatabaseHandle.i, "DROP TABLE bsl_raw")
DatabaseUpdate(Program\DatabaseHandle.i, "DROP TABLE class_of_station")
DatabaseUpdate(Program\DatabaseHandle.i, "DROP TABLE class_of_station_raw")
DatabaseUpdate(Program\DatabaseHandle.i, "DROP TABLE client")
DatabaseUpdate(Program\DatabaseHandle.i, "DROP TABLE client_raw")
DatabaseUpdate(Program\DatabaseHandle.i, "DROP TABLE client_type")
DatabaseUpdate(Program\DatabaseHandle.i, "DROP TABLE client_type_raw")
DatabaseUpdate(Program\DatabaseHandle.i, "DROP TABLE device_details")
DatabaseUpdate(Program\DatabaseHandle.i, "DROP TABLE device_details_raw")
DatabaseUpdate(Program\DatabaseHandle.i, "DROP TABLE fee_status")
DatabaseUpdate(Program\DatabaseHandle.i, "DROP TABLE fee_status_raw")
DatabaseUpdate(Program\DatabaseHandle.i, "DROP TABLE industry_cat")
DatabaseUpdate(Program\DatabaseHandle.i, "DROP TABLE industry_cat_raw")
DatabaseUpdate(Program\DatabaseHandle.i, "DROP TABLE licence")
DatabaseUpdate(Program\DatabaseHandle.i, "DROP TABLE licence_raw")
DatabaseUpdate(Program\DatabaseHandle.i, "DROP TABLE licence_service")
DatabaseUpdate(Program\DatabaseHandle.i, "DROP TABLE licence_service_raw")
DatabaseUpdate(Program\DatabaseHandle.i, "DROP TABLE licence_status")
DatabaseUpdate(Program\DatabaseHandle.i, "DROP TABLE licence_status_raw")
DatabaseUpdate(Program\DatabaseHandle.i, "DROP TABLE licence_subservice")
DatabaseUpdate(Program\DatabaseHandle.i, "DROP TABLE licence_subservice_raw")
DatabaseUpdate(Program\DatabaseHandle.i, "DROP TABLE licensing_area")
DatabaseUpdate(Program\DatabaseHandle.i, "DROP TABLE licensing_area_raw")
DatabaseUpdate(Program\DatabaseHandle.i, "DROP TABLE MAP_TILE_STORE")
DatabaseUpdate(Program\DatabaseHandle.i, "DROP TABLE nature_of_service")
DatabaseUpdate(Program\DatabaseHandle.i, "DROP TABLE nature_of_service_raw")
DatabaseUpdate(Program\DatabaseHandle.i, "DROP TABLE reports_text_block")
DatabaseUpdate(Program\DatabaseHandle.i, "DROP TABLE reports_text_block_raw")
DatabaseUpdate(Program\DatabaseHandle.i, "DROP TABLE satellite")
DatabaseUpdate(Program\DatabaseHandle.i, "DROP TABLE satellite_raw")
DatabaseUpdate(Program\DatabaseHandle.i, "DROP TABLE saved_queries")
DatabaseUpdate(Program\DatabaseHandle.i, "DROP TABLE site")
DatabaseUpdate(Program\DatabaseHandle.i, "DROP TABLE site_raw")
DatabaseUpdate(Program\DatabaseHandle.i, "DROP TABLE tmp_site_asl")
DatabaseUpdate(Program\DatabaseHandle.i, "DROP TABLE __WebKitDatabaseInfoTable__")
DatabaseUpdate(Program\DatabaseHandle.i, "VACUUM")
Else
Debug "The database table 'composite' failed to be created"
EndIf
Else
Debug "No Callsigns were found in the ACMA database, programmer error"
EndIf
Else
Debug "The requested database failed to be opened"
EndIf
End