diff --git a/docs/samples/BSBDirectoryFtp.json b/docs/samples/BSBDirectoryFtp.json new file mode 100644 index 0000000..f83a737 --- /dev/null +++ b/docs/samples/BSBDirectoryFtp.json @@ -0,0 +1,418 @@ +{ + "content-type": "application/vnd.martilq.json", + "title": "SampleFetchBSB", + "uid": "08e22a4d-5a11-4b19-9172-feb36601009e", + "description": "", + "modified": "2021-10-23", + "publisher": "meerkat", + "contactPoint": "", + "accessLevel": "Public", + "rights": "Publish", + "tags": ["BSB", "Australia"], + "license": "", + "state": "active", + "batch": 1.0, + "describedBy": "", + "landingPage": "", + "theme": "", + "resources": [ + { + "title": "BSBDirectoryAug21-305", + "uid": "59f1c10e-a71b-45a4-9205-2b651cf04304", + "documentName": "BSBDirectoryAug21-305.csv", + "issuedDate": "2021-10-23T18:37:58", + "modified": "2021-10-23T18:37:46", + "state": "active", + "author": "Meerkat@merebox.com", + "length": 1352016, + "hash": { + "algo": "SHA256", + "value": "26e40760989ff8a314cda70628c38b66d0d565ea8ad677b81b83780e33984104", + "signed": false + }, + "description": "", + "url": "ftp://bsb.hostedftp.com/~auspaynetftp/BSB/BSBDirectoryAug21-305.csv", + "version": "", + "content-type": "text/csv", + "compression": null, + "encryption": null, + "attributes": [ + { + "category": "dataset", + "name": "header", + "function": "count", + "comparison": "NA", + "value": 1 + }, + { + "category": "dataset", + "name": "footer", + "function": "count", + "comparison": "NA", + "value": 0 + }, + { + "category": "format", + "name": "separator", + "function": "value", + "comparison": "NA", + "value": "," + }, + { + "category": "format", + "name": "columns", + "function": "value", + "comparison": "NA", + "value": "," + }, + { + "category": "dataset", + "name": "records", + "function": "count", + "comparison": "EQ", + "value": 15055 + }, + { + "category": "dataset", + "name": "columns", + "function": "count", + "comparison": "EQ", + "value": 8 + } + ] + }, + { + "title": "BSBDirectoryAug21-305", + "uid": "cd79772f-ff33-4fb6-b90e-94118f80fd69", + "documentName": "BSBDirectoryAug21-305.txt", + "issuedDate": "2021-10-23T18:37:58", + "modified": "2021-10-23T18:37:48", + "state": "active", + "author": "Meerkat@merebox.com", + "length": 1701441, + "hash": { + "algo": "SHA256", + "value": "799f937f33b0284f0493063f51117f154c415a9554b8ba7d07f47416d75208a3", + "signed": false + }, + "description": "", + "url": "ftp://bsb.hostedftp.com/~auspaynetftp/BSB/BSBDirectoryAug21-305.txt", + "version": "", + "content-type": "text/plain", + "compression": null, + "encryption": null, + "attributes": [ + { + "category": "dataset", + "name": "header", + "function": "count", + "comparison": "NA", + "value": 1 + }, + { + "category": "dataset", + "name": "footer", + "function": "count", + "comparison": "NA", + "value": 0 + }, + { + "category": "format", + "name": "separator", + "function": "value", + "comparison": "NA", + "value": "," + }, + { + "category": "format", + "name": "columns", + "function": "value", + "comparison": "NA", + "value": "," + }, + { + "category": "dataset", + "name": "records", + "function": "count", + "comparison": "EQ", + "value": 15057 + }, + { + "category": "dataset", + "name": "columns", + "function": "count", + "comparison": "EQ", + "value": 1 + } + ] + }, + { + "title": "BSBDirectoryJul21-304", + "uid": "4f57b82a-6b6a-4612-8d4e-67289b7795e1", + "documentName": "BSBDirectoryJul21-304.csv", + "issuedDate": "2021-10-23T18:37:58", + "modified": "2021-10-23T18:37:51", + "state": "active", + "author": "Meerkat@merebox.com", + "length": 1351065, + "hash": { + "algo": "SHA256", + "value": "5306b8baaafaf60600b645f3514dde3f654ac2b0bdbae96b0d0501b18488b1b5", + "signed": false + }, + "description": "", + "url": "ftp://bsb.hostedftp.com/~auspaynetftp/BSB/BSBDirectoryJul21-304.csv", + "version": "", + "content-type": "text/csv", + "compression": null, + "encryption": null, + "attributes": [ + { + "category": "dataset", + "name": "header", + "function": "count", + "comparison": "NA", + "value": 1 + }, + { + "category": "dataset", + "name": "footer", + "function": "count", + "comparison": "NA", + "value": 0 + }, + { + "category": "format", + "name": "separator", + "function": "value", + "comparison": "NA", + "value": "," + }, + { + "category": "format", + "name": "columns", + "function": "value", + "comparison": "NA", + "value": "," + }, + { + "category": "dataset", + "name": "records", + "function": "count", + "comparison": "EQ", + "value": 15044 + }, + { + "category": "dataset", + "name": "columns", + "function": "count", + "comparison": "EQ", + "value": 8 + } + ] + }, + { + "title": "BSBDirectoryJul21-304", + "uid": "e8bbdfc4-ab46-465c-af97-39e4e4a3e36c", + "documentName": "BSBDirectoryJul21-304.txt", + "issuedDate": "2021-10-23T18:37:58", + "modified": "2021-10-23T18:37:53", + "state": "active", + "author": "Meerkat@merebox.com", + "length": 1700198, + "hash": { + "algo": "SHA256", + "value": "60d14b4ae28c8e91499c73e385504ea904134c0fa348af25a7baf58eb9eb8390", + "signed": false + }, + "description": "", + "url": "ftp://bsb.hostedftp.com/~auspaynetftp/BSB/BSBDirectoryJul21-304.txt", + "version": "", + "content-type": "text/plain", + "compression": null, + "encryption": null, + "attributes": [ + { + "category": "dataset", + "name": "header", + "function": "count", + "comparison": "NA", + "value": 1 + }, + { + "category": "dataset", + "name": "footer", + "function": "count", + "comparison": "NA", + "value": 0 + }, + { + "category": "format", + "name": "separator", + "function": "value", + "comparison": "NA", + "value": "," + }, + { + "category": "format", + "name": "columns", + "function": "value", + "comparison": "NA", + "value": "," + }, + { + "category": "dataset", + "name": "records", + "function": "count", + "comparison": "EQ", + "value": 15046 + }, + { + "category": "dataset", + "name": "columns", + "function": "count", + "comparison": "EQ", + "value": 1 + } + ] + }, + { + "title": "BSBDirectorySep21-306", + "uid": "08d11601-54a6-4fb0-b694-e2d2c2a7eb32", + "documentName": "BSBDirectorySep21-306.csv", + "issuedDate": "2021-10-23T18:37:58", + "modified": "2021-10-23T18:37:55", + "state": "active", + "author": "Meerkat@merebox.com", + "length": 1351281, + "hash": { + "algo": "SHA256", + "value": "4f5c97799f6006247fad64f9a5acc425e79f31342cbbbdaaba2995af16ac56b4", + "signed": false + }, + "description": "", + "url": "ftp://bsb.hostedftp.com/~auspaynetftp/BSB/BSBDirectorySep21-306.csv", + "version": "", + "content-type": "text/csv", + "compression": null, + "encryption": null, + "attributes": [ + { + "category": "dataset", + "name": "header", + "function": "count", + "comparison": "NA", + "value": 1 + }, + { + "category": "dataset", + "name": "footer", + "function": "count", + "comparison": "NA", + "value": 0 + }, + { + "category": "format", + "name": "separator", + "function": "value", + "comparison": "NA", + "value": "," + }, + { + "category": "format", + "name": "columns", + "function": "value", + "comparison": "NA", + "value": "," + }, + { + "category": "dataset", + "name": "records", + "function": "count", + "comparison": "EQ", + "value": 15050 + }, + { + "category": "dataset", + "name": "columns", + "function": "count", + "comparison": "EQ", + "value": 8 + } + ] + }, + { + "title": "BSBDirectorySep21-306", + "uid": "a4a01312-a254-4ba9-b9c7-294b1589a2e1", + "documentName": "BSBDirectorySep21-306.txt", + "issuedDate": "2021-10-23T18:37:58", + "modified": "2021-10-23T18:37:58", + "state": "active", + "author": "Meerkat@merebox.com", + "length": 1700876, + "hash": { + "algo": "SHA256", + "value": "dd830996c087b5b781a8d4d2bf7ec8da2b0de642a338ffbadea0d8d3834e2232", + "signed": false + }, + "description": "", + "url": "ftp://bsb.hostedftp.com/~auspaynetftp/BSB/BSBDirectorySep21-306.txt", + "version": "", + "content-type": "text/plain", + "compression": null, + "encryption": null, + "attributes": [ + { + "category": "dataset", + "name": "header", + "function": "count", + "comparison": "NA", + "value": 1 + }, + { + "category": "dataset", + "name": "footer", + "function": "count", + "comparison": "NA", + "value": 0 + }, + { + "category": "format", + "name": "separator", + "function": "value", + "comparison": "NA", + "value": "," + }, + { + "category": "format", + "name": "columns", + "function": "value", + "comparison": "NA", + "value": "," + }, + { + "category": "dataset", + "name": "records", + "function": "count", + "comparison": "EQ", + "value": 15052 + }, + { + "category": "dataset", + "name": "columns", + "function": "count", + "comparison": "EQ", + "value": 1 + } + ] + } + ], + "custom": [ + { + "extension": "software", + "softwareName": "MARTILQREFERENCE", + "author": "Meerkat@merebox.com", + "version": "0.0.1" + } + ] +} \ No newline at end of file diff --git a/docs/samples/BSBDirectoryHttp.json b/docs/samples/BSBDirectoryHttp.json new file mode 100644 index 0000000..444ac61 --- /dev/null +++ b/docs/samples/BSBDirectoryHttp.json @@ -0,0 +1,223 @@ +{ + "content-type": "application/vnd.martilq.json", + "title": "SampleFetchBSB", + "uid": "08e22a4d-5a11-4b19-9172-feb36601009e", + "description": "", + "modified": "2021-10-23", + "publisher": "meerkat", + "contactPoint": "", + "accessLevel": "Public", + "rights": "Publish", + "tags": ["BSB", "Australia"], + "license": "", + "state": "active", + "batch": 1.0, + "describedBy": "", + "landingPage": "", + "theme": "", + "resources": [ + { + "title": "BSBDirectoryAug21-305", + "uid": "59f1c10e-a71b-45a4-9205-2b651cf04304", + "documentName": "BSBDirectoryAug21-305.csv", + "issuedDate": "2021-10-23T18:37:58", + "modified": "2021-10-23T18:37:46", + "state": "active", + "author": "Meerkat@merebox.com", + "length": 1352016, + "hash": { + "algo": "SHA256", + "value": "26e40760989ff8a314cda70628c38b66d0d565ea8ad677b81b83780e33984104", + "signed": false + }, + "description": "", + "url": "http://apnedata.merebox.com.s3.ap-southeast-2.amazonaws.com/au/bsb/BSBDirectoryAug21-305.csv", + "version": "", + "content-type": "text/csv", + "compression": null, + "encryption": null, + "attributes": [ + { + "category": "dataset", + "name": "header", + "function": "count", + "comparison": "NA", + "value": 1 + }, + { + "category": "dataset", + "name": "footer", + "function": "count", + "comparison": "NA", + "value": 0 + }, + { + "category": "format", + "name": "separator", + "function": "value", + "comparison": "NA", + "value": "," + }, + { + "category": "format", + "name": "columns", + "function": "value", + "comparison": "NA", + "value": "," + }, + { + "category": "dataset", + "name": "records", + "function": "count", + "comparison": "EQ", + "value": 15055 + }, + { + "category": "dataset", + "name": "columns", + "function": "count", + "comparison": "EQ", + "value": 8 + } + ] + }, + { + "title": "BSBDirectoryJul21-304", + "uid": "4f57b82a-6b6a-4612-8d4e-67289b7795e1", + "documentName": "BSBDirectoryJul21-304.csv", + "issuedDate": "2021-10-23T18:37:58", + "modified": "2021-10-23T18:37:51", + "state": "active", + "author": "Meerkat@merebox.com", + "length": 1351065, + "hash": { + "algo": "SHA256", + "value": "5306b8baaafaf60600b645f3514dde3f654ac2b0bdbae96b0d0501b18488b1b5", + "signed": false + }, + "description": "", + "url": "http://apnedata.merebox.com.s3.ap-southeast-2.amazonaws.com/au/bsb/BSBDirectoryJul21-304.csv", + "version": "", + "content-type": "text/csv", + "compression": null, + "encryption": null, + "attributes": [ + { + "category": "dataset", + "name": "header", + "function": "count", + "comparison": "NA", + "value": 1 + }, + { + "category": "dataset", + "name": "footer", + "function": "count", + "comparison": "NA", + "value": 0 + }, + { + "category": "format", + "name": "separator", + "function": "value", + "comparison": "NA", + "value": "," + }, + { + "category": "format", + "name": "columns", + "function": "value", + "comparison": "NA", + "value": "," + }, + { + "category": "dataset", + "name": "records", + "function": "count", + "comparison": "EQ", + "value": 15044 + }, + { + "category": "dataset", + "name": "columns", + "function": "count", + "comparison": "EQ", + "value": 8 + } + ] + }, + { + "title": "BSBDirectorySep21-306", + "uid": "08d11601-54a6-4fb0-b694-e2d2c2a7eb32", + "documentName": "BSBDirectorySep21-306.csv", + "issuedDate": "2021-10-23T18:37:58", + "modified": "2021-10-23T18:37:55", + "state": "active", + "author": "Meerkat@merebox.com", + "length": 1351281, + "hash": { + "algo": "SHA256", + "value": "4f5c97799f6006247fad64f9a5acc425e79f31342cbbbdaaba2995af16ac56b4", + "signed": false + }, + "description": "", + "url": "http://apnedata.merebox.com.s3.ap-southeast-2.amazonaws.com/au/bsb/BSBDirectorySep21-306.csv", + "version": "", + "content-type": "text/csv", + "compression": null, + "encryption": null, + "attributes": [ + { + "category": "dataset", + "name": "header", + "function": "count", + "comparison": "NA", + "value": 1 + }, + { + "category": "dataset", + "name": "footer", + "function": "count", + "comparison": "NA", + "value": 0 + }, + { + "category": "format", + "name": "separator", + "function": "value", + "comparison": "NA", + "value": "," + }, + { + "category": "format", + "name": "columns", + "function": "value", + "comparison": "NA", + "value": "," + }, + { + "category": "dataset", + "name": "records", + "function": "count", + "comparison": "EQ", + "value": 15050 + }, + { + "category": "dataset", + "name": "columns", + "function": "count", + "comparison": "EQ", + "value": 8 + } + ] + } + ], + "custom": [ + { + "extension": "software", + "softwareName": "MARTILQREFERENCE", + "author": "Meerkat@merebox.com", + "version": "0.0.1" + } + ] +} \ No newline at end of file diff --git a/docs/samples/powershell/Invoke-SampleGenerateBsb.ps1 b/docs/samples/powershell/Invoke-SampleGenerateBsb.ps1 index 21f9a4a..e3b63c0 100644 --- a/docs/samples/powershell/Invoke-SampleGenerateBsb.ps1 +++ b/docs/samples/powershell/Invoke-SampleGenerateBsb.ps1 @@ -152,7 +152,7 @@ foreach($file in Get-ChildItem $localDirectory) } } $oResource = New-MartiResource -SourcePath $zipFile -UrlPath $localDirectory -LogPath ".\test\Logs" -ExtendAttributes -Set-AttributeValueString -Attributes $oResource.attributes -Key "compression" -Category "format" -Function "algorithm" -Value "WINZIP" +Set-AttributeValueString -Attributes $oResource.attributes -Key "compression" -Category "format" -Function "algo" -Value "WINZIP" $oMarti.resources += $oResource $fileJson = Join-Path -Path $localDirectory -ChildPath "MartiBSBZip.mti" @@ -224,7 +224,7 @@ Compress-7Zip -Path $noticeFile -ArchiveFileName $zipFile -Append -Password $sec $oResource = New-MartiResource -SourcePath $zipFile -UrlPath $localDirectory -LogPath ".\test\Logs" -ExtendAttributes $oResource.compression = "7ZIP" $oResource.encryption = New-Encryption -Algorithm "Passphrase" -Value $secret -Set-AttributeValueString -Attributes $oResource.attributes -Key "compression" -Category "format" -Function "algorithm" -Value "7ZIP" +Set-AttributeValueString -Attributes $oResource.attributes -Key "compression" -Category "format" -Function "algo" -Value "7ZIP" $oMarti.resources += $oResource $fileJson = Join-Path -Path $localDirectory -ChildPath "MartiBSBSecure.mti" diff --git a/docs/samples/powershell/Invoke-SampleGenerateBsbSecure.ps1 b/docs/samples/powershell/Invoke-SampleGenerateBsbSecure.ps1 index 7bb4e98..dcaf236 100644 --- a/docs/samples/powershell/Invoke-SampleGenerateBsbSecure.ps1 +++ b/docs/samples/powershell/Invoke-SampleGenerateBsbSecure.ps1 @@ -161,7 +161,7 @@ $oResource.encryption = New-Encryption -Algorithm "PKI" -Value $($encryptedSecre Write-Debug "Secret: $secret" -Set-AttributeValueString -Attributes $oResource.attributes -Key "compression" -Category "format" -Function "algorithm" -Value "7ZIP" +Set-AttributeValueString -Attributes $oResource.attributes -Key "compression" -Category "format" -Function "algo" -Value "7ZIP" $oMarti.resources += $oResource $fileJson = Join-Path -Path $localDirectory -ChildPath "MartiBSB7ZipPKI.mti" diff --git a/docs/samples/python/SampleFetchFtpBsb.py b/docs/samples/python/SampleFetchFtpBsb.py new file mode 100644 index 0000000..cd3127e --- /dev/null +++ b/docs/samples/python/SampleFetchFtpBsb.py @@ -0,0 +1,56 @@ + +import os +import sys +import csv + +sys.path.insert(0, "../../../source/python/client") +from martiLQ import * + +test_directory = "./test/fetch_ftp" +os.environ["MARTILQ_LOGPATH"] = os.path.join(test_directory, "logs") + +if not os.path.exists("./test"): + os.mkdir("./test") + +if not os.path.exists(test_directory): + os.mkdir(test_directory) + +print("Creating martiLQ definition") +mlq = martiLQ() +mlq.LoadConfig() +print("Loading definition json") +mlq.Load("../BSBDirectoryFtp.json") +print("Fetching files") +fetched_files, fetch_error = mlq.Fetch(test_directory) + +if len(fetched_files) < 0: + raise Exception("No resource files fetched") +else: + print("Fetched {} files".format(len(fetched_files))) + +if len(fetch_error) > 0: + raise Exception("Some resources not fetched") + +print("Generate the self value, overriding existing") +oMarti = mlq.NewMartiDefinition() +for full_fileName in fetched_files: + if os.path.isfile(full_fileName): + oResource = mlq.NewMartiLQResource(full_fileName, "", False, True) + oMarti["resources"].append(oResource) +print("Perform validation test") +lqresults, testError = mlq.TestMartiDefinition("../BSBDirectoryFtp.json") + +testfile = open("./test/LoadQualityTest_Ftp.csv", "w+", newline ="") +with testfile: + lqwriter = csv.writer(testfile) + lqwriter.writerows(lqresults) + +if testError > 0: + print("MISMATCH DETECTED") +else: + print("RECONCILED") + +mlq.CloseLog() + +print("Test completed: SampleFetchFtpBsb.py") + diff --git a/docs/samples/python/SampleFetchHttpBsb.py b/docs/samples/python/SampleFetchHttpBsb.py new file mode 100644 index 0000000..c60bf9b --- /dev/null +++ b/docs/samples/python/SampleFetchHttpBsb.py @@ -0,0 +1,56 @@ + +import os +import sys +import csv + +sys.path.insert(0, "../../../source/python/client") +from martiLQ import * + +test_directory = "./test/fetch_http" +os.environ["MARTILQ_LOGPATH"] = os.path.join(test_directory, "logs") + +if not os.path.exists("./test"): + os.mkdir("./test") + +if not os.path.exists(test_directory): + os.mkdir(test_directory) + +print("Creating martiLQ definition") +mlq = martiLQ() +mlq.LoadConfig() +print("Loading definition json") +mlq.Load("../BSBDirectoryHttp.json") +print("Fetching files") +fetched_files, fetch_error = mlq.Fetch(test_directory) + +if len(fetched_files) < 0: + raise Exception("No resource files fetched") +else: + print("Fetched {} files".format(len(fetched_files))) + +if len(fetch_error) > 0: + raise Exception("Some resources not fetched") + +print("Generate the self value, overriding existing") +oMarti = mlq.NewMartiDefinition() +for full_fileName in fetched_files: + if os.path.isfile(full_fileName): + oResource = mlq.NewMartiLQResource(full_fileName, "", False, True) + oMarti["resources"].append(oResource) +print("Perform validation test") +lqresults, testError = mlq.TestMartiDefinition("../BSBDirectoryHttp.json") + +testfile = open("./test/LoadQualityTest_Http.csv", "w+", newline ="") +with testfile: + lqwriter = csv.writer(testfile) + lqwriter.writerows(lqresults) + +if testError > 0: + print("MISMATCH DETECTED") +else: + print("RECONCILED") + +mlq.CloseLog() + +print("Test completed: SampleFetchHttpBsb.py") + diff --git a/docs/samples/python/SampleGenerateBsb.py b/docs/samples/python/SampleGenerateBsb.py index daff6b6..e4c7671 100644 --- a/docs/samples/python/SampleGenerateBsb.py +++ b/docs/samples/python/SampleGenerateBsb.py @@ -4,11 +4,15 @@ import os import json import sys import csv +import zipfile + sys.path.insert(0, "../../../source/python/client") -#from source.python.client.martiLQ import martiLQ from martiLQ import * +ftpFetch = True +os.environ["MARTILQ_LOGPATH"] = "./test/logs" + def ftpList(host, path): files = [] @@ -58,7 +62,8 @@ for file_name in files: if file_name.endswith(".csv") | file_name.endswith(".txt"): file_remote = remote_dir + file_name file_local = "./test/" + file_name -# ftpPull(remote_host, file_remote, file_local) + if ftpFetch: + ftpPull(remote_host, file_remote, file_local) print("Creating martiLQ definition") mlq = martiLQ() @@ -67,16 +72,56 @@ oMarti = mlq.NewMartiDefinition() for file_name in files: if file_name.startswith("BSBDirectory"): if file_name.endswith(".csv") | file_name.endswith(".txt"): - oResource = mlq.NewMartiResource(os.path.join("./test/", file_name), "", False, True, "./test/logs") + oResource = mlq.NewMartiLQResource(os.path.join("./test/", file_name), "", False, True) oMarti["resources"].append(oResource) mlq.CloseLog() -print("Save martiLQ definition") -jd = json.dumps(oMarti, indent=5) + +print("Save martiLQ definition") jsonFile = open("./test/BSBDirectoryPlain.mti", "w") -jsonFile.write(jd) +jsonFile.write(json.dumps(oMarti, indent=5)) jsonFile.close() +print("Base sample mti written: BSBDirectoryPlain.mti") + + +print("Creating martiLQ ZIP file") +zipFileName = "BSBDirectory.zip" +fileZipCount = 0 + +mlq = martiLQ() +oMarti = mlq.NewMartiDefinition() +with zipfile.ZipFile("./test/" + zipFileName, "w", compression=zipfile.ZIP_DEFLATED) as zipObj: + for file_name in files: + if file_name.startswith("BSBDirectory"): + if file_name.endswith(".csv") | file_name.endswith(".txt"): + file_remote = remote_dir + file_name + file_local = "./test/" + file_name + if ftpFetch: + ftpPull(remote_host, file_remote, file_local) + zipObj.write(file_local, file_name) + fileZipCount = fileZipCount + 1 + oResource = mlq.NewMartiLQResource(os.path.join("./test/", file_name), "", False, True) + oResource["url"] = "@"+zipFileName + "/" + file_name + oMarti["resources"].append(oResource) + + +oResource = mlq.NewMartiLQResource(os.path.join("./test/", zipFileName), "", False, True) +oResource["url"] = "./test/" + zipFileName +mlq.SetAttributeValueString(Attributes=oResource["attributes"], Key="compression", Category="format", Function="algo", Value="WINZIP") +mlq.SetAttributeValueNumber(Attributes=oResource["attributes"], Key="files", Category="dataset", Function="count", Value=fileZipCount) +oMarti["resources"].append(oResource) + +mlq.CloseLog() + +print("Save martiLQ ZIP definition") +jsonFile = open("./test/MartiLQ_BSBZip.mti", "w") +jsonFile.write(json.dumps(oMarti, indent=5)) +jsonFile.close() +print("ZIP sample mti written: MartiLQ_BSBZip.mti") + + + print("Sample completed: SampleGenerateBsb.py") lqresults, testError = mlq.TestMartiDefinition("./test/BSBDirectoryPlain.mti") diff --git a/docs/samples/python/SampleGenerateBsbSecure.py b/docs/samples/python/SampleGenerateBsbSecure.py new file mode 100644 index 0000000..aa48be2 --- /dev/null +++ b/docs/samples/python/SampleGenerateBsbSecure.py @@ -0,0 +1,92 @@ + +import ftplib +import os +import json +import sys +import csv + +sys.path.insert(0, "../../../source/python/client") +from martiLQ import * + +def ftpList(host, path): + + files = [] + with ftplib.FTP(host) as ftp: + try: + ftp.login() + ftp.cwd(path) + files = ftp.nlst() + except ftplib.all_errors as e: + print('FTP error:', e) + + return files + + +def ftpPull(host, file_remote, file_local): + + with ftplib.FTP(host) as ftp: + + try: + ftp.login() + + with open(file_local, 'wb') as fl: + res = ftp.retrbinary(f"RETR {file_remote}", fl.write) + if not res.startswith('226 Transfer complete'): + print('Download failed') + if os.path.isfile(file_local): + os.remove(file_local) + + except ftplib.all_errors as e: + print('FTP error:', e) + if os.path.isfile(file_local): + os.remove(file_local) + + +remote_host = "bsb.hostedftp.com" +remote_dir = "/~auspaynetftp/BSB/" + +print("Fetch sample file list") +files = ftpList(remote_host, remote_dir) + +if not os.path.exists("./test"): + os.mkdir("./test") + +print("Fetch sample files") +for file_name in files: + if file_name.startswith("BSBDirectory"): + if file_name.endswith(".csv") | file_name.endswith(".txt"): + file_remote = remote_dir + file_name + file_local = "./test/" + file_name + ftpPull(remote_host, file_remote, file_local) + +print("Creating martiLQ definition") +mlq = martiLQ() +oMarti = mlq.NewMartiDefinition() + +for file_name in files: + if file_name.startswith("BSBDirectory"): + if file_name.endswith(".csv") | file_name.endswith(".txt"): + oResource = mlq.NewMartiLQResource(os.path.join("./test/", file_name), "", False, True, "./test/logs") + oMarti["resources"].append(oResource) + +mlq.CloseLog() +print("Save martiLQ definition") +jd = json.dumps(oMarti, indent=5) + +jsonFile = open("./test/BSBDirectoryPlain.mti", "w") +jsonFile.write(jd) +jsonFile.close() +print("Sample completed: SampleGenerateBsb.py") + +lqresults, testError = mlq.TestMartiDefinition("./test/BSBDirectoryPlain.mti") + +testfile = open("./test/LoadQualityTest01.csv", "w+", newline ="") +with testfile: + lqwriter = csv.writer(testfile) + lqwriter.writerows(lqresults) + +if testError: + print("MISMATCH DETECTED") + +print("Test completed: SampleGenerateBsb.py") + diff --git a/source/java/README.md b/source/java/README.md index 4536e7d..4f74a34 100644 --- a/source/java/README.md +++ b/source/java/README.md @@ -1,3 +1,12 @@ -# Place maker +# Java code for martiLQ +The ``client`` folder contains Java library for creating +**martiLQ** definition and comparison for the Load Quality. +See the Java samples directory on how to use the code. + +__TO COME__ + +Publish of Java library to ``Maven`` is yet to occur + +This will occur after merge of code into ``main`` branch diff --git a/source/powershell/Compare-MartiResource.ps1 b/source/powershell/Compare-MartiLQResource.ps1 similarity index 100% rename from source/powershell/Compare-MartiResource.ps1 rename to source/powershell/Compare-MartiLQResource.ps1 diff --git a/source/powershell/MartiLQItem.ps1 b/source/powershell/MartiLQItem.ps1 index b8f9b01..e55b8be 100644 --- a/source/powershell/MartiLQItem.ps1 +++ b/source/powershell/MartiLQItem.ps1 @@ -144,7 +144,7 @@ Param( Get-ChildItem $SourceFolder -Filter $Filter -Recurse:$Recurse -Force| Where-Object {!$_.PSIsContainer} | ForEach-Object { - $oResource = New-MartiResource -SourcePath $_.FullName -UrlPath $remoteDirectory -LogPath $LogPath -ExtendAttributes:$ExtendAttributes -ExcludeHash:$ExcludeHash + $oResource = New-MartiResource -SourcePath $_.FullName -UrlPath $UrlPath -LogPath $LogPath -ExtendAttributes:$ExtendAttributes -ExcludeHash:$ExcludeHash if ($null -ne $UrlPath -and $UrlPath -ne "") { $postfixName = $_.FullName.Replace($SourceFullName, "") @@ -289,7 +289,7 @@ function New-DefaultZipAttributes { $oAttribute = [PSCustomObject]@{ category = "format" name = "compression" - function = "algorithm" + function = "algo" comparison = "NA" value = $CompressionType } @@ -298,7 +298,7 @@ function New-DefaultZipAttributes { $oAttribute = [PSCustomObject]@{ category = "format" name = "encryption" - function = "algorithm" + function = "algo" comparison = "NA" value = $Encryption } diff --git a/source/powershell/README.md b/source/powershell/README.md index 4536e7d..d0e6304 100644 --- a/source/powershell/README.md +++ b/source/powershell/README.md @@ -1,3 +1,13 @@ -# Place maker +# PowerShell code for martiLQ +This folder contains PowerShell module for creating +**martiLQ** definition and comparison for the Load Quality. + +See the PowerShell samples directory on how to use the code. + +__TO COME__ + +Publish of PowerShell code as ``Install-Module martiLQ`` is yet to occur + +This will occur after merge of code into ``main`` branch diff --git a/source/python/README.md b/source/python/README.md index 4536e7d..d222d53 100644 --- a/source/python/README.md +++ b/source/python/README.md @@ -1,3 +1,12 @@ -# Place maker +# Python code for martiLQ +The ``client`` folder contains Python module for creating +**martiLQ** definition and comparison for the Load Quality. +See the Python samples directory on how to use the code. + +__TO COME__ + +Publish of Python code as ``pip install martiLQ`` is yet to occur + +This will occur after merge of code into ``main`` branch diff --git a/source/python/client/martiLQ.py b/source/python/client/martiLQ.py index 6efbe01..b531aa0 100644 --- a/source/python/client/martiLQ.py +++ b/source/python/client/martiLQ.py @@ -8,47 +8,172 @@ import json import datetime import getpass import hashlib - +import glob +from configparser import ConfigParser +import requests +import mimetypes class martiLQ: - gLogPathName = "" - gSoftwareVersion = "0.0.1" - gdefault_metaFile = "##marti##.mti" + _SoftwareVersion = "0.0.1" + _default_metaFile = "##marti##.mti" - gMartiErrorId = "" - gLogOpen = False + _oSoftware = { + "extension": "software", + "softwareName": "MARTILQREFERENCE", + "author": "Meerkat@merebox.com", + "version": "0.0.1" + } - oMartiDefinition = None - def __init__(self): - self.gLogOpen = False + _MartiErrorId = "" + _LogOpen = False - def Get(self): - return self.oMartiDefinition + _oMartiDefinition = None + + _oConfiguration = None - def Close(self): - if self.gLogOpen: - self.CloseLog() - self.gLogOpen = False def GetSoftwareName(self): return "MARTILQREFERENCE" + def __init__(self): + self._LogOpen = False + + _oSoftware = { + "extension": "software", + "softwareName": self.GetSoftwareName(), + "author": "Meerkat@merebox.com", + "version": self._SoftwareVersion + } + + self._oConfiguration = { + "softwareName": self.GetSoftwareName(), + "author": "Meerkat@merebox.com", + "version": self._SoftwareVersion, + + "logPath": None, + + "state": "active", + "accessLevel": "Confidential", + "rights": "Restricted", + + "hashAlgorithm": "SHA256", + "signKey_File": None, + "signKey_Password": None, + + "loaded": False + } + + def LoadConfig(self, ConfigPath=None): + + config_object = ConfigParser() + if not ConfigPath is None: + if os.path.exists(ConfigPath): + config_object.read(ConfigPath) + else: + self.WriteLog("Configuration path '{}' does not exist".format(ConfigPath)) + raise Exception("Configuration path '{}' does not exist".format(ConfigPath)) + else: + # Look in default location and name + home = os.path.expanduser('~') + if os.path.exists(os.path.join(home, ".martilq/martilq.ini")): + ConfigPath = os.path.join(home, ".martilq/martilq.ini") + if os.path.exists("martilq.ini"): + ConfigPath = "martilq.ini" + if not ConfigPath is None: + self.WriteLog("Usig configuration path '{}'".format(ConfigPath)) + config_object.read(ConfigPath) + + if config_object.has_section("Resources"): + items = config_object["Resource"] + if not items is None: + config_attr = ["accessLevel", "rights", "state"] + for x in config_attr: + if not items[x] is None and not items[x] == "": + self._oConfiguration[x] = items[x] + + if config_object.has_section("Hash"): + items = config_object["Hash"] + if not items is None: + config_attr = ["hashAlgorithm", "signKey_File", "signKey_Password"] + for x in config_attr: + if not items[x] is None and not items[x] == "": + self._oConfiguration[x] = items[x] + + # Now check environmental values + self._oConfiguration["signKey_File"] = os.getenv("MARTILQ_SIGNKEY_FILE", self._oConfiguration["signKey_File"]) + self._oConfiguration["signKey_Password"] = os.getenv("MARTILQ_SIGNKEY_PASSWORD", self._oConfiguration["signKey_Password"]) + self._oConfiguration["logPath"] = os.getenv("MARTILQ_LOGPATH", self._oConfiguration["logPath"]) + + self.WriteLog("Configuration load processed") + + def Set(self, MartiLQ): + self._oMartiDefinition = MartiLQ + + def Get(self): + return self._oMartiDefinition + + def Save(self, JsonPath): + jsonFile = open(JsonPath, "w") + jsonFile.write(json.dumps(self._oMartiDefinition, indent=5)) + jsonFile.close() + + def Load(self, JsonPath): + + self._MartiErrorId = "" + + self.OpenLog() + self.WriteLog("Function 'Load' parameters follow") + self.WriteLog("Parameter: SourcePath Value: {}".format(JsonPath)) + self.WriteLog("") + + if os.path.exists(JsonPath): + self.WriteLog("Overwriting existing definition") + else: + if not os.path.exists(os.path.dirname(JsonPath)): + self.WriteLog("Parent folder does not exist") + raise Exception("Parent folder '{}' does not exist".format(os.path.dirname(JsonPath))) + + if not self._oMartiDefinition is None: + self.WriteLog("Existing definition overwritten") + + jsonFile = open(JsonPath, "r") + self._oMartiDefinition = json.load(jsonFile) + jsonFile.close() + + + def SetConfig(self, Key=None, Value=None): + + if not Key is None: + self._oConfiguration[Key] = Value + + def GetConfig(self, Key=None): + + if not Key is None: + return self._oConfiguration[Key] + else: + return None + + def Close(self): + if self._LogOpen: + self.CloseLog() + self._LogOpen = False + def GetLogName(self): today = datetime.datetime.today() dateToday = today.strftime("%Y-%m-%d") - if None == self.gLogPathName or self.gLogPathName == "": + if None == self._oConfiguration["logPath"] or self._oConfiguration["logPath"] == "": return None - if not os.path.exists(self.gLogPathName): - os.mkdir(self.gLogPathName) + if not os.path.exists(self._oConfiguration["logPath"]): + os.mkdir(self._oConfiguration["logPath"]) logName = self.GetSoftwareName() + "_" + dateToday + ".log" - return os.path.join(self.gLogPathName, logName) + return os.path.join(self._oConfiguration["logPath"], logName) def WriteLog(self, LogEntry): @@ -75,55 +200,48 @@ class martiLQ: def OpenLog(self): - if not self.gLogOpen: + if not self._LogOpen: today = datetime.datetime.today() dateToday = today.strftime("%Y-%m-%d") self.WriteLog("***********************************************************************************") self.WriteLog("* Start of processing: {}".format(dateToday)) self.WriteLog("***********************************************************************************") - self.gLogOpen = True + self._LogOpen = True def CloseLog(self): - if self.gLogOpen: + if self._LogOpen: today = datetime.datetime.today() dateToday = today.strftime("%Y-%m-%d") self.WriteLog("***********************************************************************************") self.WriteLog("* End of processing: {}".format(dateToday)) self.WriteLog("***********************************************************************************") - self.gLogOpen = False + self._LogOpen = False def NewMartiDefinition(self): today = datetime.datetime.today() dateToday = today.strftime("%Y-%m-%d") - oSoftware = { - "extension": "software", - "softwareName": self.GetSoftwareName(), - "author": "Meerkat@merebox.com", - "version": self.gSoftwareVersion - } - publisher = getpass.getuser() lcustom = [] - lcustom.append(oSoftware) + lcustom.append(self._oSoftware) lresource = [] - self.oMartiDefinition = { + self._oMartiDefinition = { + "content-type": "application/vnd.martilq.json", "title": "", "uid": str(uuid.uuid4()), - "resources": lresource, "description": "", "modified": dateToday, - "tags": ["document", self.GetSoftwareName()], "publisher": publisher, "contactPoint": "", - "accessLevel": "Confidential", - "rights": "Restricted", + "accessLevel": self.GetConfig("accessLevel"), + "rights": self.GetConfig("rights"), + "tags": [], "license": "", "state": "active", "batch": 1.0, @@ -131,23 +249,54 @@ class martiLQ: "landingPage": "", "theme": "", + "resources": lresource, "custom": lcustom } - return self.oMartiDefinition + return self._oMartiDefinition + def NewMartiChildItem(self, SourceFolder, UrlPath=None, Recurse=True, ExtendAttributes=True, ExcludeHash=False, Filter ="*"): - def NewMartiResource(self, SourcePath, UrlPath, ExcludeHash, ExtendAttributes, LogPath): + SourceFullName = os.path.abspath(SourceFolder) + + for fullName in glob.iglob(SourceFullName, recursive=Recurse): + if os.path.isfile(fullName): + oResource = self.NewMartiLQResource(SourcePath=fullName, UrlPath=UrlPath, ExtendAttributes=ExtendAttributes, ExcludeHash=ExcludeHash) + self._oMartiDefinition["resources"].append(oResource) + + + def GetContentType(self, SourcePath): + + ext = None + + # Some overrides + match = str(os.path.splitext(SourcePath)[1][1:]).lower() + if (ext is None or ext == "") and match == "csv": + ext = "text/csv" + + if ext is None or ext == "": + ext, _ = mimetypes.guess_type(SourcePath, strict=True) + + if ext is None or ext == "": + match = str(os.path.splitext(SourcePath)[1][1:]).lower() + if match == "md": + ext = "application/markdown" + + if ext is None or ext == "": + ext = "application/vnd.unknown." + os.path.splitext(SourcePath)[1][1:] + + return ext + + def NewMartiLQResource(self, SourcePath, UrlPath, ExcludeHash, ExtendAttributes): today = datetime.datetime.today() - dateToday = today.strftime("%Y-%m-%d") + dateToday = today.strftime("%Y-%m-%dT%H:%M:%S") - self.gMartiErrorId = "" - self.gLogPathName = LogPath + self._MartiErrorId = "" self.OpenLog() - self.WriteLog("Function 'NewMartiResource' parameters follow") + self.WriteLog("Function 'NewMartiLQResource' parameters follow") self.WriteLog("Parameter: SourcePath Value: {}".format(SourcePath)) self.WriteLog("Parameter: UrlPath Value: {}".format(UrlPath)) self.WriteLog("Parameter: ExcludeHash Value: {}".format(ExcludeHash)) @@ -157,20 +306,20 @@ class martiLQ: item = os.path.basename(SourcePath) self.WriteLog("Define file {}".format(SourcePath)) - HashAlgorithm = "SHA256" + HashAlgorithm = self.GetConfig("hashAlgorithm") try: mtime = os.path.getmtime(SourcePath) except OSError: mtime = 0 - last_modified_date = datetime.datetime.fromtimestamp(mtime).strftime("%Y-%m-%d %H:%M:%S") + last_modified_date = datetime.datetime.fromtimestamp(mtime).strftime("%Y-%m-%dT%H:%M:%S") if ExcludeHash: hash = None else: - hash = self.NewMartiLQHash(Algorithm=HashAlgorithm, FilePath=SourcePath, Value="") + hash = self.NewMartiLQHash(Algorithm=HashAlgorithm, FilePath=SourcePath, Value="", Sign=self.GetConfig("signKey_File")) - lattribute = self.SetMartiResourceAttributes(SourcePath, os.path.splitext(SourcePath)[1][1:], ExtendAttributes) + lattribute = self.SetMartiLQResourceAttributes(SourcePath, str(os.path.splitext(SourcePath)[1][1:]).lower(), ExtendAttributes) oResource = { "title": item.replace(os.path.splitext(SourcePath)[1], ""), @@ -178,15 +327,15 @@ class martiLQ: "documentName": item, "issuedDate": dateToday, "modified": last_modified_date, - "state": "active", - "author": "", + "state": self.GetConfig("state"), + "author": self.GetConfig("author"), "length": os.path.getsize(SourcePath), "hash": hash, "description": "", "url": "", - "version": self.gSoftwareVersion, - "format": os.path.splitext(SourcePath)[1][1:], + "version": "", + "content-type": self.GetContentType(SourcePath), "compression": None, "encryption": None, @@ -194,7 +343,7 @@ class martiLQ: } if None != UrlPath and UrlPath != "": - if UrlPath[UrlPath.Length-1] == "/" or UrlPath[UrlPath.Length-1] == "\\": + if UrlPath[len(UrlPath)-1] == "/" or UrlPath[len(UrlPath)-1] == "\\": oResource["url"] = UrlPath.replace("\\", "/") + item else: oResource["url"] = UrlPath.replace("\\", "/") + "/" + item @@ -202,41 +351,75 @@ class martiLQ: self.WriteLog("Complete file {}".format(SourcePath)) else: - self.gMartiErrorId = "MRI2001" + self._MartiErrorId = "MRI2001" message = "Document '{}' not found or is a folder".format(SourcePath) - self.WriteLog(message + " " + self.gMartiErrorId) + self.WriteLog(message + " " + self._MartiErrorId) raise Exception(message) return oResource - def NewMartiLQHash(self, Algorithm, FilePath, Value=""): + def NewMartiLQHash(self, Algorithm, FilePath, Value="", Sign=""): - if Value == "" and FilePath != "": - if Algorithm == "SHA256": - sha_hash = hashlib.sha256() - with open(FilePath,"rb") as fh: - for byte_block in iter(lambda: fh.read(4096),b""): - sha_hash.update(byte_block) - Value = sha_hash.hexdigest() - if Algorithm == "SHA512": - sha_hash = hashlib.sha512() - with open(FilePath,"rb") as fh: - for byte_block in iter(lambda: fh.read(4096),b""): - sha_hash.update(byte_block) - Value = sha_hash.hexdigest() - if Algorithm == "MD5": - sha_hash = hashlib.md5() - with open(FilePath,"rb") as fh: - for byte_block in iter(lambda: fh.read(4096),b""): - sha_hash.update(byte_block) - Value = sha_hash.hexdigest() + try: + signed = False + if Value == "" and FilePath != "": - oHash = { - "algo": Algorithm, - "value": Value - } + if not Sign is None and not Sign == "" and not os.path.exists(Sign): + self.WriteLog("Sign file '{}' not found".format(Sign)) + Sign = "" + + if Sign is None or Sign == "": + + if Algorithm == "SHA256": + sha_hash = hashlib.sha256() + with open(FilePath,"rb") as fh: + for byte_block in iter(lambda: fh.read(4096),b""): + sha_hash.update(byte_block) + Value = sha_hash.hexdigest() + if Algorithm == "SHA512": + sha_hash = hashlib.sha512() + with open(FilePath,"rb") as fh: + for byte_block in iter(lambda: fh.read(4096),b""): + sha_hash.update(byte_block) + Value = sha_hash.hexdigest() + if Algorithm == "MD5": + sha_hash = hashlib.md5() + with open(FilePath,"rb") as fh: + for byte_block in iter(lambda: fh.read(4096),b""): + sha_hash.update(byte_block) + Value = sha_hash.hexdigest() + + else: + + import OpenSSL + from OpenSSL import crypto + import base64 + + private_key_file = open(Sign, "r") + privkey = private_key_file.read() + private_key_file.close() + password = self.GetConfig("sigenKey_Password") + + if privkey.startswith('-----BEGIN '): + pkey = crypto.load_privatekey(crypto.FILETYPE_PEM, privkey, password.encode('utf-8')) + else: + pkey = crypto.load_pkcs12(privkey, password).get_privatekey() + with open(FilePath,"rb") as fh: + sign = OpenSSL.crypto.sign(pkey, fh.read(), Algorithm) + Value = (base64.b64encode(sign)).decode('utf-8') + signed = True + + oHash = { + "algo": Algorithm, + "value": Value, + "signed": signed + } + + except Exception as e: + self.WriteLog("Hash error for file {}: {}".format(FilePath, str(e))) + raise e return oHash @@ -251,10 +434,6 @@ class martiLQ: return oEncryption - - - - def SetMartiAttribute(self, Attributes, ACategory, AName, AFunction, Comparison, Value): matched = False @@ -396,7 +575,7 @@ class martiLQ: oAttribute = { "category": "format", "name": "compression", - "function": "algorithm", + "function": "algo", "comparison": "NA", "value": CompressionType } @@ -405,7 +584,7 @@ class martiLQ: oAttribute = { "category": "format", "name": "encryption", - "function": "algorithm", + "function": "algo", "comparison": "NA", "value": Encryption } @@ -423,7 +602,7 @@ class martiLQ: return lattribute - def SetAttributeValueString(self, Attributes,Category,Key,Function,Value,Comparison = "EQ"): + def SetAttributeValueString(self, Attributes, Category, Key, Function, Value, Comparison="EQ"): for item in Attributes: @@ -447,7 +626,7 @@ class martiLQ: return - def SetAttributeValueNumber(self, Attributes,Category,Key,Function,Value,Comparison = "EQ"): + def SetAttributeValueNumber(self, Attributes, Category, Key, Function, Value, Comparison = "EQ"): for item in Attributes: @@ -472,7 +651,7 @@ class martiLQ: return - def SetMartiResourceAttributes(self, PathFile, FileType, ExtendedAttributes): + def SetMartiLQResourceAttributes(self, PathFile, FileType, ExtendedAttributes): lattribute = None @@ -526,10 +705,12 @@ class martiLQ: if FileType == "zip": lattribute = self.NewDefaultZipAttributes("ZIP") if ExtendedAttributes: - self.SetAttributeValueNumber(lattribute, "dataset", "files", "count", -1) + self.SetAttributeValueNumber(lattribute, "dataset", "files", "count", 0, Comparison="NA") if FileType == "7z": lattribute = self.NewDefaultZipAttributes("7Z") + if ExtendedAttributes: + self.SetAttributeValueNumber(lattribute, "dataset", "files", "count", 0, Comparison="NA") if lattribute == None: lattribute = [] @@ -537,6 +718,81 @@ class martiLQ: return lattribute + def FtpPull(self, host, file_remote, file_local): + + with ftplib.FTP(host) as ftp: + + try: + ftp.login() + + with open(file_local, 'wb') as fl: + res = ftp.retrbinary(f"RETR {file_remote}", fl.write) + if not res.startswith('226 Transfer complete'): + print('Download failed') + if os.path.isfile(file_local): + os.remove(file_local) + + except ftplib.all_errors as e: + print('FTP error:', e) + if os.path.isfile(file_local): + os.remove(file_local) + + + def Fetch(self, TargetPath): + + if TargetPath is None or TargetPath == "": + self.WriteLog("Target path is missing from fetch") + raise Exception("Target path is missing from fetch") + + if self._oMartiDefinition is None: + self.WriteLog("No defintion loaded") + raise Exception("No defintion loaded") + + if len(self._oMartiDefinition["resources"]) < 1: + self.WriteLog("No resources in defintion") + raise Exception("No resources in defintion") + + fetched_files = [] + fetch_error = [] + + for resource in self._oMartiDefinition["resources"]: + + if not resource["url"] is None and not resource["url"] == "": + method = str(resource["url"].split(":", 2)[0]).lower() + #print("Method of fetch {} for {}".format(method, resource["url"])) + matched = False + if method == "ftp": + matched = True + parts = resource["url"].split("/", 3) + host = parts[2] + file_remote = parts[3] + self.FtpPull(host, file_remote, os.path.join(TargetPath, resource["documentName"])) + fetched_files.append(os.path.join(TargetPath, resource["documentName"])) + + if method == "http" or method == "https": + matched = True + response = requests.get(resource["url"]) + if not response.status_code == 200: + self.WriteLog("HTP fetch failed with code {} for '{}'".format(response.status_code, resource["url"])) + print("HTP fetch failed with code {} for '{}'".format(response.status_code, resource["url"])) + fetch_error.append(resource["url"]) + else: + with open(os.path.join(TargetPath, resource["documentName"]),'wb') as fh: + fh.write(response.content) + fetched_files.append(os.path.join(TargetPath, resource["documentName"])) + + if method == "file": + pass + + if not matched: + fetch_error.append(resource["documentName"]) + + else: + fetch_error.append(resource["documentName"]) + + return fetched_files, fetch_error + + def TestAttributeDefinition(self, oTestResults, documentName, localR, remoteR): errorCount = 0 @@ -564,57 +820,88 @@ class martiLQ: return errorCount - def TestMartiDefinition(self, LQSourcePath, oMarti = None, LogPath =""): + def TestMartiDefinition(self, SourcePath, Sign=None): - self.gMartiErrorId = "" - self.gLogPathName = LogPath + self._MartiErrorId = "" self.OpenLog() self.WriteLog("Function 'TestMartiDefinition' parameters follow") - self.WriteLog("Parameter: oMarti Value: {}".format(oMarti)) - self.WriteLog("Parameter: LQSourcePath Value: {}".format(LQSourcePath)) + self.WriteLog("Parameter: SourcePath Value: {}".format(SourcePath)) self.WriteLog("") - if oMarti is None: - oMarti = self.oMartiDefinition - - if oMarti is None: + if self._oMartiDefinition is None: pass - if not os.path.exists(LQSourcePath): + if not os.path.exists(SourcePath): pass - jsonFile = open(LQSourcePath, "r") + jsonFile = open(SourcePath, "r") lq = json.load(jsonFile) jsonFile.close() - testError = False + testError = 0 oTestResults = [] otest = ["ResourceName", "Level", "Metric", "Matches", "LocalCalculation", "SuppliedValue" ] oTestResults.append(otest) - otest = ["", "Batch", "Resource count", (len(oMarti["resources"]) == len(lq["resources"])), len(oMarti["resources"]), len(lq["resources"]) ] + otest = ["", "Batch", "Resource count", (len(self._oMartiDefinition["resources"]) == len(lq["resources"])), len(self._oMartiDefinition["resources"]), len(lq["resources"]) ] oTestResults.append(otest) - for resource in oMarti["resources"]: + for resource in self._oMartiDefinition["resources"]: for retarget in lq["resources"]: if resource["documentName"] == retarget["documentName"]: - testError = testError or resource["hash"]["value"] != retarget["hash"]["value"] - otest = [resource["documentName"], "Resource", "Hash", (resource["hash"]["value"] == retarget["hash"]["value"]), resource["hash"]["value"], retarget["hash"]["value"] ] - oTestResults.append(otest) + if retarget["hash"]["signed"]: + # Need to verify the hash + if Sign is None: + Sign = self.GetConfig("signKey_file") - testError = testError or resource["length"] != retarget["length"] + if Sign is None: + self.WriteLog("No Sign Key specified so Hash check cannot be performed for signed content") + else: + try: + import OpenSSL + from OpenSSL import crypto + import base64 + except ImportError: + self.WriteLog("Import error in signed verification") + + pub_key_file = open(Sign, "r") + pubkey = pub_key_file.read() + pub_key_file.close() + + x509 = crypto.X509() + x509.set_pubkey(pubkey) + + try: + crypto.verify(x509, retarget["hash"]["value"], resource["hash"]["value"], retarget["hash"]["algo"]) + otest = [resource["documentName"], "Resource", "Hash",False, resource["hash"]["value"], retarget["hash"]["value"] ] + except: + testError = testError + 1 + self.WriteLog("Error in verification for {}".format(resource["documentName"])) + otest = [resource["documentName"], "Resource", "Hash", True, resource["hash"]["value"], retarget["hash"]["value"] ] + + oTestResults.append(otest) + + pass + else: + if not resource["hash"]["value"] == retarget["hash"]["value"]: + testError = testError + 1 + otest = [resource["documentName"], "Resource", "Hash", (resource["hash"]["value"] == retarget["hash"]["value"]), resource["hash"]["value"], retarget["hash"]["value"] ] + oTestResults.append(otest) + + if not resource["length"] == retarget["length"]: + testError = testError + 1 otest = [resource["documentName"], "Resource", "Length", (resource["length"] == retarget["length"]), resource["length"], retarget["length"] ] oTestResults.append(otest) errorAttrCount = self.TestAttributeDefinition(oTestResults, resource["documentName"], resource["attributes"], retarget["attributes"]) - - if errorAttrCount > 0: - testError = True + testError = testError + errorAttrCount break + self.WriteLog("TestMartiDefinition function completed with {} errors".format(testError)) + return oTestResults, testError diff --git a/test/powershell/test_Marti.ps1 b/test/powershell/test_Marti.ps1 deleted file mode 100644 index 07dc7e9..0000000 --- a/test/powershell/test_Marti.ps1 +++ /dev/null @@ -1,44 +0,0 @@ - -. .\source\powershell\MartiLQ.ps1 -. .\source\powershell\Compress-MartiLQ.ps1 - -Write-Host "Test case #1" -$oMarti = New-MartiChildItem -SourceFolder ".\docs" -Recurse -UrlPath ".\docs" -Filter "*" -LogPath ".\test\powershell\results\Logs" -$oMarti.description = "Sample execution" - -$x = ConvertTo-Json -InputObject $oMarti -Set-Content -Path ".\test\powershell\results\marti_test01.mti" -Value $x - -Write-Host "Test case #2" -$ArchiveFile = ".\test\powershell\results\marti_test02.zip" -Compress-MartiLQ -SourceFolder ".\docs" -Filter "*" -LogPath ".\test\powershell\results\Logs" -ArchiveFile $ArchiveFile - -Write-Host "Test case #3" -$y = Get-MartiItem -MartiDefintiion $oMarti -Title "ckan" -Format "txt" -LogPath ".\test\powershell\results\Logs" -Write-Host "Get item Title: $($y.title)" -Write-Host "Get item Url: $($y.url)" - -Write-Host "Test case #4" -$oMarti = New-MartiResource -SourcePath ".\docs\ckan.md" -LogPath ".\test\powershell\results\Logs" -$oMarti.description = "Sample execution for ckan" - -$x = ConvertTo-Json -InputObject $oMarti -Set-Content -Path ".\test\powershell\results\marti_test02json.mti" -Value $x - -$x = ConvertTo-Csv -InputObject $oMarti -Set-Content -Path ".\test\powershell\results\marti_test02csv.mti." -Value $x - -$x = ConvertTo-Xml -As String -InputObject $oMarti -Depth 6 -Set-Content -Path ".\test\powershell\results\marti_test02xml.mti" -Value $x - -$x = ConvertTo-Html -InputObject $oMarti -Set-Content -Path ".\test\powershell\results\marti_test02html.mti" -Value $x - -Write-Host "Test case #5" -$oMarti = New-MartiResource -SourcePath ".\docs\eror" -LogPath ".\test\powershell\results\Logs" -$oMarti.description = "Sample execution with error" - -$x = ConvertTo-Json -InputObject $oMarti -Set-Content -Path ".\test\powershell\results\marti_test03.mti" -Value $x - - diff --git a/test/powershell/test_MartiData1.ps1 b/test/powershell/test_MartiData1.ps1 deleted file mode 100644 index 86c8b95..0000000 --- a/test/powershell/test_MartiData1.ps1 +++ /dev/null @@ -1,96 +0,0 @@ - -. .\source\powershell\New-Marti.ps1 -. .\source\powershell\ConvertFrom-Ckan.ps1 - -if (!(Test-Path -Path ".\test\powershell\results\data")) { - $null = New-Item -Path ".\test\powershell\results\data" -ItemType Directory -} - -#$x = Get-Content -Path ".\test\results\data\bsb.csv" -$bsbFile = ".\test\powershell\results\data\bsb.csv" -$data = Import-Csv -Path $bsbFile - -$columns = ($data | get-member -type NoteProperty).count -$rows = @($data).count - -Write-Host "Rows: $rows Columns: $columns" - -[System.Collections.ArrayList]$lattribute = @() - -$oAttribute = [PSCustomObject]@{ - category = "dataset" - name = "header" - function = "count" - comparison = "EQ" - value = 0 -} - -$lattribute += $oAttribute - -$oAttribute = [PSCustomObject]@{ - category = "dataset" - name = "footer" - function = "count" - comparison = "EQ" - value = 0 -} - -$lattribute += $oAttribute - -$oAttribute = [PSCustomObject]@{ - category = "dataset" - name = "rows" - function = "count" - comparison = "EQ" - value = $rows -} - -$lattribute += $oAttribute - -$oAttribute = [PSCustomObject]@{ - category = "dataset" - name = "columns" - function = "count" - comparison = "EQ" - value = $columns -} - -$lattribute += $oAttribute - -$oAttribute = [PSCustomObject]@{ - category = "data" - name = "BSB" - function = "sum" - comparison = "EQ" - value = 1032092 -} - -$lattribute += $oAttribute - -$oAttribute = [PSCustomObject]@{ - category = "data" - name = "BSB" - function = "unique" - comparison = "EQ" - value = $rows -} - -$lattribute += $oAttribute - - - -$uq = Get-Content $bsbFile | ConvertFrom-Csv -Header "C1", "C2" | Select-Object "C2" | Sort-Object "C2" -Unique | Group-Object -Property "C2" -$oAttribute = [PSCustomObject]@{ - category = "data" - name = "Institution" - function = "unique" - comparison = "EQ" - value = $uq.Count -} - -$lattribute += $oAttribute - -$x = ConvertTo-Json -InputObject $lattribute -$x -#select -skip 6 -#| select name diff --git a/test/powershell/test_MartiData2.ps1 b/test/powershell/test_MartiData2.ps1 deleted file mode 100644 index 47f649f..0000000 --- a/test/powershell/test_MartiData2.ps1 +++ /dev/null @@ -1,41 +0,0 @@ - -. .\source\powershell\New-Marti.ps1 -. .\source\powershell\ConvertFrom-Ckan.ps1 -. .\source\powershell\Compare-MartiResource.ps1 - - -$bsbFile = ".\test\powershell\results\data\bsb.csv" - -Write-Host ">>>>>>Test case #1" -$x = New-MartiResource -SourcePath $bsbFile -UrlPath "" -ExcludeHash -LogPath ".\test\powershell\results\Logs" - -Write-Host ">>>>>>Test case #2" -$x.resources - -[System.Collections.ArrayList] $Attr = Set-MartiAttribute -Attributes $x.resources[0].attributes -ACategory "dataset" -AName "records" -AFunction "count" -comparison "EQ" -value 10 -$x.resources[0].attributes = Set-MartiAttribute -Attributes $Attr -ACategory "dataset" -AName "columns" -AFunction "count" -comparison "EQ" -value 8 - -Write-Host ">>>>>>Test case #3" -$x.resources[0].attributes - -Write-Host ">>>>>>Test case #4" -$y = Compare-MartiResource -DataSource $bsbFile -Resource $x.resources[0] -LogPath ".\test\powershell\results\Logs" -$y - -$covidFile = ".\test\powershell\results\data\covid-case-locations-20210920-1315.json" - -Write-Host ">>>>>>Test case #5" -$x = New-MartiResource -SourcePath $covidFile -UrlPath "" -ExcludeHash -LogPath ".\test\powershell\results\Logs" - -Write-Host ">>>>>>Test case #6" -$x.resources - -[System.Collections.ArrayList] $Attr = Set-MartiAttribute -Attributes $x.resources[0].attributes -ACategory "dataset" -AName "records" -AFunction "count" -comparison "EQ" -value 10 -$x.resources[0].attributes = Set-MartiAttribute -Attributes $Attr -ACategory "dataset" -AName "columns" -AFunction "count" -comparison "EQ" -value 8 - -Write-Host ">>>>>>Test case #7" -$x.resources[0].attributes - -Write-Host ">>>>>>Test case #8" -$y = Compare-MartiResource -DataSource $covidFile -Resource $x.resources[0] -LogPath ".\test\powershell\results\Logs" -$y diff --git a/test/powershell/test_MartiData3.ps1 b/test/powershell/test_MartiData3.ps1 deleted file mode 100644 index 9e7f2e1..0000000 --- a/test/powershell/test_MartiData3.ps1 +++ /dev/null @@ -1,18 +0,0 @@ - -. .\source\powershell\New-Marti.ps1 -. .\source\powershell\ConvertFrom-Ckan.ps1 -. .\source\powershell\Compare-MartiResource.ps1 - - -$covidFile = ".\test\powershell\results\data\covid-case-locations-20210920-1315.json" - -$x = New-MartiResource -SourcePath $covidFile -UrlPath "" -ExcludeHash -LogPath ".\test\powershell\results\Logs" - -[System.Collections.ArrayList] $attr = Set-MartiAttribute -Attributes $x.resources[0].attributes -ACategory "dataset" -AName "records" -AFunction "count" -comparison "EQ" -value 516 -$attr = Set-MartiAttribute -Attributes $attr -ACategory "dataset" -AName "columns" -AFunction "count" -comparison "EQ" -value 12 -$x.resources[0].attributes = Set-MartiAttribute -Attributes $attr -ACategory "format" -AName "list" -AFunction "offset" -comparison "EQ" -value "data.monitor" - -$y = Compare-MartiResource -DataSource $covidFile -Resource $x.resources[0] -LogPath ".\test\powershell\results\Logs" -$y - -$attr | Get-Member \ No newline at end of file diff --git a/test/powershell/test_MartiLQ.ps1 b/test/powershell/test_MartiLQ.ps1 new file mode 100644 index 0000000..91dcfe0 --- /dev/null +++ b/test/powershell/test_MartiLQ.ps1 @@ -0,0 +1,49 @@ + +. .\source\powershell\MartiLQ.ps1 +. .\source\powershell\Compress-MartiLQ.ps1 + +try { + + Write-Host "Test case #1" + $oMarti = New-MartiChildItem -SourceFolder ".\docs" -Recurse -UrlPath ".\docs" -Filter "*" -LogPath ".\test\powershell\results\Logs" + $oMarti.description = "Sample execution" + + $x = ConvertTo-Json -InputObject $oMarti + Set-Content -Path ".\test\powershell\results\marti_test01.mti" -Value $x + + Write-Host "Test case #2" + $ArchiveFile = ".\test\powershell\results\marti_test02.zip" + Compress-MartiLQ -SourceFolder ".\docs" -Filter "*" -LogPath ".\test\powershell\results\Logs" -ArchiveFile $ArchiveFile + + Write-Host "Test case #3" + $y = Get-MartiItem -MartiDefintiion $oMarti -Title "ckan" -Format "txt" -LogPath ".\test\powershell\results\Logs" + Write-Host "Get item Title: $($y.title)" + Write-Host "Get item Url: $($y.url)" + + Write-Host "Test case #4" + $oMarti = New-MartiResource -SourcePath ".\docs\ckan.md" -LogPath ".\test\powershell\results\Logs" + $oMarti.description = "Sample execution for ckan" + + $x = ConvertTo-Json -InputObject $oMarti + Set-Content -Path ".\test\powershell\results\marti_test02json.mti" -Value $x + + $x = ConvertTo-Csv -InputObject $oMarti + Set-Content -Path ".\test\powershell\results\marti_test02csv.mti." -Value $x + + $x = ConvertTo-Xml -As String -InputObject $oMarti -Depth 6 + Set-Content -Path ".\test\powershell\results\marti_test02xml.mti" -Value $x + + $x = ConvertTo-Html -InputObject $oMarti + Set-Content -Path ".\test\powershell\results\marti_test02html.mti" -Value $x + + Write-Host "Test case #5" + $oMarti = New-MartiResource -SourcePath ".\docs\eror" -LogPath ".\test\powershell\results\Logs" + $oMarti.description = "Sample execution with error" + + $x = ConvertTo-Json -InputObject $oMarti + Set-Content -Path ".\test\powershell\results\marti_test03.mti" -Value $x + +} +catch { + throw +} diff --git a/test/powershell/test_MartiCkan.ps1 b/test/powershell/test_MartiLQCkan.ps1 similarity index 91% rename from test/powershell/test_MartiCkan.ps1 rename to test/powershell/test_MartiLQCkan.ps1 index 9f3d995..cdaeee6 100644 --- a/test/powershell/test_MartiCkan.ps1 +++ b/test/powershell/test_MartiLQCkan.ps1 @@ -1,6 +1,6 @@ -. .\source\powershell\New-Marti.ps1 +. .\source\powershell\MartiLQ.ps1 . .\source\powershell\ConvertFrom-Ckan.ps1 @@ -23,7 +23,6 @@ Set-Content -Path ".\test\powershell\results\marti_test06.mti" -Value $x # cases $covid19 = "https://data.nsw.gov.au/data/api/3/action/package_show?id=3dc5dc39-40b4-4ee9-8ec6-2d862a916dcf" -#Invoke-WebRequest $covid19 -Method GET -OutFile ".\test\powershell\results\data\nsw_covid19.csv" $covid_2 = Invoke-WebRequest $covid19 $oMarti = ConvertFrom-Ckan -InputObject $covid_2 $oMarti.description = "This data has been converted from NSW CKAN data source with URL '$covid19'" diff --git a/test/powershell/test_MartiLQData1.ps1 b/test/powershell/test_MartiLQData1.ps1 new file mode 100644 index 0000000..fefd136 --- /dev/null +++ b/test/powershell/test_MartiLQData1.ps1 @@ -0,0 +1,100 @@ + +. .\source\powershell\MartiLQ.ps1 +. .\source\powershell\ConvertFrom-Ckan.ps1 + +if (!(Test-Path -Path ".\test\powershell\results\data")) { + $null = New-Item -Path ".\test\powershell\results\data" -ItemType Directory +} + +try { + + $bsbFile = ".\test\powershell\results\data\BSBDirectorySep21-306.csv" + $data = Import-Csv -Path $bsbFile + + $columns = ($data | get-member -type NoteProperty).count + $rows = @($data).count + + Write-Host "Rows: $rows Columns: $columns" + + [System.Collections.ArrayList]$lattribute = @() + + $oAttribute = [PSCustomObject]@{ + category = "dataset" + name = "header" + function = "count" + comparison = "EQ" + value = 0 + } + + $lattribute += $oAttribute + + $oAttribute = [PSCustomObject]@{ + category = "dataset" + name = "footer" + function = "count" + comparison = "EQ" + value = 0 + } + + $lattribute += $oAttribute + + $oAttribute = [PSCustomObject]@{ + category = "dataset" + name = "rows" + function = "count" + comparison = "EQ" + value = $rows + } + + $lattribute += $oAttribute + + $oAttribute = [PSCustomObject]@{ + category = "dataset" + name = "columns" + function = "count" + comparison = "EQ" + value = $columns + } + + $lattribute += $oAttribute + + $oAttribute = [PSCustomObject]@{ + category = "data" + name = "BSB" + function = "sum" + comparison = "EQ" + value = 1032092 + } + + $lattribute += $oAttribute + + $oAttribute = [PSCustomObject]@{ + category = "data" + name = "BSB" + function = "unique" + comparison = "EQ" + value = $rows + } + + $lattribute += $oAttribute + + + + $uq = Get-Content $bsbFile | ConvertFrom-Csv -Header "C1", "C2" | Select-Object "C2" | Sort-Object "C2" -Unique | Group-Object -Property "C2" + $oAttribute = [PSCustomObject]@{ + category = "data" + name = "Institution" + function = "unique" + comparison = "EQ" + value = $uq.Count + } + + $lattribute += $oAttribute + + $x = ConvertTo-Json -InputObject $lattribute + $x + +} +catch { + throw +} \ No newline at end of file diff --git a/test/powershell/test_MartiLQData2.ps1 b/test/powershell/test_MartiLQData2.ps1 new file mode 100644 index 0000000..24ba9e2 --- /dev/null +++ b/test/powershell/test_MartiLQData2.ps1 @@ -0,0 +1,48 @@ + +. .\source\powershell\MartiLQ.ps1 +. .\source\powershell\MartiLQItem.ps1 +. .\source\powershell\ConvertFrom-Ckan.ps1 +. .\source\powershell\Compare-MartiLQResource.ps1 + +try { + + $bsbFile = ".\test\powershell\results\data\BSBDirectorySep21-306.csv" + + Write-Host ">>>>>>Test case #1" + $x = New-MartiResource -SourcePath $bsbFile -UrlPath "" -ExcludeHash -LogPath ".\test\powershell\results\Logs" + + Write-Host ">>>>>>Test case #2" + $x + + [System.Collections.ArrayList] $Attr = Set-MartiAttribute -Attributes $x.attributes -ACategory "dataset" -AName "records" -AFunction "count" -comparison "EQ" -value 10 + $x.attributes = Set-MartiAttribute -Attributes $Attr -ACategory "dataset" -AName "columns" -AFunction "count" -comparison "EQ" -value 8 + + Write-Host ">>>>>>Test case #3" + $x.attributes + + Write-Host ">>>>>>Test case #4" + $y = Compare-MartiResource -DataSource $bsbFile -Resource $x -LogPath ".\test\powershell\results\Logs" + $y + + $covidFile = ".\test\powershell\results\data\covid-case-locations-20210920-1315.json" + + Write-Host ">>>>>>Test case #5" + $x = New-MartiResource -SourcePath $covidFile -UrlPath "" -ExcludeHash -LogPath ".\test\powershell\results\Logs" + + Write-Host ">>>>>>Test case #6" + $x + + [System.Collections.ArrayList] $Attr = Set-MartiAttribute -Attributes $x.attributes -ACategory "dataset" -AName "records" -AFunction "count" -comparison "EQ" -value 10 + $x.attributes = Set-MartiAttribute -Attributes $Attr -ACategory "dataset" -AName "columns" -AFunction "count" -comparison "EQ" -value 8 + + Write-Host ">>>>>>Test case #7" + $x.attributes + + Write-Host ">>>>>>Test case #8" + $y = Compare-MartiResource -DataSource $covidFile -Resource $x -LogPath ".\test\powershell\results\Logs" + $y + +} +catch { + throw +} \ No newline at end of file diff --git a/test/powershell/test_MartiLQData3.ps1 b/test/powershell/test_MartiLQData3.ps1 new file mode 100644 index 0000000..964b832 --- /dev/null +++ b/test/powershell/test_MartiLQData3.ps1 @@ -0,0 +1,25 @@ + +. .\source\powershell\MartiLQ.ps1 +. .\source\powershell\MartiLQItem.ps1 +. .\source\powershell\ConvertFrom-Ckan.ps1 +. .\source\powershell\Compare-MartiLQResource.ps1 + +try { + + $covidFile = ".\test\powershell\results\data\covid-case-locations-20210920-1315.json" + + $x = New-MartiResource -SourcePath $covidFile -UrlPath "" -ExcludeHash -LogPath ".\test\powershell\results\Logs" + + [System.Collections.ArrayList] $attr = Set-MartiAttribute -Attributes $x.attributes -ACategory "dataset" -AName "records" -AFunction "count" -comparison "EQ" -value 516 + $attr = Set-MartiAttribute -Attributes $attr -ACategory "dataset" -AName "columns" -AFunction "count" -comparison "EQ" -value 12 + $x.attributes = Set-MartiAttribute -Attributes $attr -ACategory "format" -AName "list" -AFunction "offset" -comparison "EQ" -value "data.monitor" + + $y = Compare-MartiResource -DataSource $covidFile -Resource $x -LogPath ".\test\powershell\results\Logs" + $y + + $attr | Get-Member + +} +catch { + throw +} \ No newline at end of file diff --git a/test/private.pem b/test/private.pem new file mode 100644 index 0000000..2f040da --- /dev/null +++ b/test/private.pem @@ -0,0 +1,30 @@ +-----BEGIN RSA PRIVATE KEY----- +Proc-Type: 4,ENCRYPTED +DEK-Info: DES-EDE3-CBC,43D90161F4DF49CC + +RLyZTiOgXqyrj8/ecWIKf2oZviplxQCqfhr9s7VfuvPlQwDVroi91d9AnrJvOCFL +yyhHTIQxDj0NoEIZ+DDToo2qvoHlmStIpYs1WXvFRZnVsA2KUPmcg9vaRlTb4n4i +XU9c/uWDDvdI2k+BuFoeyWaPnn9gBS9y7sPcvBwpDHMMVQ9Quv5q74c9O99Rtb/p +gpKWAEO6fLRw8xGyaMnoGCMQbcaa/af6hSuvnFkrJwlkp9fYyfz9bWwwznjwsRBT +QC+T5pTp5ZgQ7J/d0Kr21RrBFUlUDtw7C3y5UCY54ajAPuHpoBEXutOGNbzuHNFq +sA1i9iu7L2TBT2gp4XlCtnq9RDqAirn/heNDlzJWYAi00a4yratmQepq4n1M3L1e +MMb61PNQckvyeWjwLc1QoxK+spyxd9ZjriEXTv7/UjnNgfWXep3KqR4QNRO6u6xq +7Bg/87kY1hpLHNLgVhp6laKe68pvxeO3X0OMKw1HOGI9CxdYi92GYXsNgQULscnp +ukusKVPHQOUCQe0YzkVcIjrmJ3/EMRvJ3cXRvxmZfal2mJNOsuBwEQunkJRwDBKR +WpBArhNRToGG6gCe4+i9vXzyXjYg+szs8XyONY0AzTW+8HXEuBddp+otke7fSsYp +QIYCoinNjnXyaTuNHIlFl/9/FXacjaBAS+5UVEbYZ80oYPzcqMhbaUl2be9UDxQn +ThjnYoUo+zYGrCqvqVLf/uTM0Wgd/Ac3jZlS86vsHmYi2LXjsWRdSuD11w25pB9r +hSrt2n6hRavB8XxOoni6K0lBKTOI4KyxVFDFD2e5aUj2pYpOkG+sjjGM/YQw16Uo +yA1IiiOUaBGWZ/EnFhHLD9oeOyQ5LD3F6A2IHSfEYK0JSZhlZKzL1GGrlrgGtY3O +gYKijUhjZpGRqlTKO7jG3v6HzQia4sw1mi0C8IlNXZIGqa6HcaCidi3CsFdcKoiB +bye0xIIQJPAMR0ybi6ptRAKjINGGlIc1pAfxWq4lu4MDp9zvpo52GaHxvSapaye8 +JnXDLCfa5gxBBr4WOz7S49gb3UP+6JZUGGKTWsd/RdZNDqmR9GMlTjVx6anMLEwQ +YvWyqbPfuzHXe6+XCDYinBTv35eTbZC28FpwvBXniZ8grM0jxqSAi24KQk6f0Ynx +KtWwgGsQLy+5BpX1PZAGRnKZBGWOsjOAOgvn1ZWTEzhu8woci6On8ZtgXuLmemws +uQdjDCl7B7/fHIElPMg/3FZkYg4cTooazNvBnszbvubkkNFbVFFLn/e3RMSsAuJy +FURqYoXp3DkcF1931oJeBjno3MblgQ/6nmWrt/Z6PHd6FVbJqeEEExDWdtz5nWvf +BmFK/TFhrn8zeUGbxtZjUhHGTHC9a36ybeFsc04i8yE/8vbpYguEHGBPndro6xpK +xIEkZV/EfAgwbu3HiRlVCgLFWyKGz+flG6A1pMeKsbeFmdz4WieuFOjxQtL5Qhek +obyPH3XFWv5xFD3TxxMgRKFUXsoWCjAHE5RXi7F0+YJd9GeKqFutohHJyCOFc6L8 +PZ8b6e7wkh8jqOKwJnlZXDYooI5jvXNWr+KJb96zYWYEHGIkdmvmdg== +-----END RSA PRIVATE KEY----- diff --git a/test/public.pem b/test/public.pem new file mode 100644 index 0000000..40a2a15 --- /dev/null +++ b/test/public.pem @@ -0,0 +1,9 @@ +-----BEGIN PUBLIC KEY----- +MIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEAmpZRUYQql7FCIPB67IbS +jB15jDRkVARdIrzdnfEx3rXG3l5YI2+oilN8sLaS+oot/PYNQICOHOe1w6hhWnzI +eviX2PgA1bf9lNDHYSN3R70F2b1CbrFIA/WXNv58s/yjg5aIjD0lfQ/o1KjAIsUc +cA8VKkW8sB0SidjDdHSsyWWF0LKiRGFIAfLgE/spG06zT4M1fB9AgWmqrcCOwW6s +DC5dFXnW5ZnaAD5xD8gZdAM/i7JybCOXOJKZLiQNXGLMYp1iFk8fSeuiyJRVIgqI +mIYbbPkKaYNBIi9LmUg67HcSODL8uZDtogiRpIZV72NjhsK4iGWvoY9EcKaMNMgS +cwIDAQAB +-----END PUBLIC KEY----- diff --git a/test/python/test_MartiLQ1.py b/test/python/test_MartiLQ1.py new file mode 100644 index 0000000..6e6665a --- /dev/null +++ b/test/python/test_MartiLQ1.py @@ -0,0 +1,39 @@ + +import os +import json +import sys +import csv +import zipfile + +sys.path.insert(0, "./source/python/client") +from martiLQ import * + +os.environ["MARTILQ_LOGPATH"] = "./test/python/results/logs" + +print("Python test case #1") + +mlq = martiLQ() +mlq.LoadConfig() +oMarti = mlq.NewMartiDefinition() +mlq.NewMartiChildItem(SourceFolder= "./docs/*", UrlPath="./docs" , ExcludeHash=False, ExtendAttributes=True) + +oMarti["description"] = "Sample execution #1" + +print("Save martiLQ definition #1") +mlq.Save("./test/python/results/DocsPlain1.mti") + +print("Save martiLQ definition #2") +oMarti["description"] = "Sample execution #2" +jsonFile = open("./test/python/results/DocsPlain2.mti", "w") +jsonFile.write(json.dumps(oMarti, indent=5)) +jsonFile.close() +print("Base sample mti written: DocsPlain2.mti") + +print("Load martiLQ definition #1") +mlq.Load("./test/python/results/DocsPlain1.mti") +oMarti = mlq.Get() +print("Definition description is: {}".format(oMarti["description"])) + +mlq.CloseLog() + +print("Completed Python test case #1")