Wardrobe
/
martiLQ
mirror of https://github.com/meerkat-manor/martiLQ.git
3
0
Fork 0
Code Issues Projects Releases Wiki Activity

Changed name to MartiLQ with refactor 

Added more functionality
draft_specifications
meerkat 2021-10-23 22:48:57 +11:00
parent 463328245b
commit ed84028bd4
26 changed files with 1629 additions and 324 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

418
docs/samples/BSBDirectoryFtp.json 100644
View File

@ -0,0 +1,418 @@
{
"content-type": "application/vnd.martilq.json",
"title": "SampleFetchBSB",
"uid": "08e22a4d-5a11-4b19-9172-feb36601009e",
"description": "",
"modified": "2021-10-23",
"publisher": "meerkat",
"contactPoint": "",
"accessLevel": "Public",
"rights": "Publish",
"tags": ["BSB", "Australia"],
"license": "",
"state": "active",
"batch": 1.0,
"describedBy": "",
"landingPage": "",
"theme": "",
"resources": [
{
"title": "BSBDirectoryAug21-305",
"uid": "59f1c10e-a71b-45a4-9205-2b651cf04304",
"documentName": "BSBDirectoryAug21-305.csv",
"issuedDate": "2021-10-23T18:37:58",
"modified": "2021-10-23T18:37:46",
"state": "active",
"author": "Meerkat@merebox.com",
"length": 1352016,
"hash": {
"algo": "SHA256",
"value": "26e40760989ff8a314cda70628c38b66d0d565ea8ad677b81b83780e33984104",
"signed": false
},
"description": "",
"url": "ftp://bsb.hostedftp.com/~auspaynetftp/BSB/BSBDirectoryAug21-305.csv",
"version": "",
"content-type": "text/csv",
"compression": null,
"encryption": null,
"attributes": [
{
"category": "dataset",
"name": "header",
"function": "count",
"comparison": "NA",
"value": 1
},
{
"category": "dataset",
"name": "footer",
"function": "count",
"comparison": "NA",
"value": 0
},
{
"category": "format",
"name": "separator",
"function": "value",
"comparison": "NA",
"value": ","
},
{
"category": "format",
"name": "columns",
"function": "value",
"comparison": "NA",
"value": ","
},
{
"category": "dataset",
"name": "records",
"function": "count",
"comparison": "EQ",
"value": 15055
},
{
"category": "dataset",
"name": "columns",
"function": "count",
"comparison": "EQ",
"value": 8
}
]
},
{
"title": "BSBDirectoryAug21-305",
"uid": "cd79772f-ff33-4fb6-b90e-94118f80fd69",
"documentName": "BSBDirectoryAug21-305.txt",
"issuedDate": "2021-10-23T18:37:58",
"modified": "2021-10-23T18:37:48",
"state": "active",
"author": "Meerkat@merebox.com",
"length": 1701441,
"hash": {
"algo": "SHA256",
"value": "799f937f33b0284f0493063f51117f154c415a9554b8ba7d07f47416d75208a3",
"signed": false
},
"description": "",
"url": "ftp://bsb.hostedftp.com/~auspaynetftp/BSB/BSBDirectoryAug21-305.txt",
"version": "",
"content-type": "text/plain",
"compression": null,
"encryption": null,
"attributes": [
{
"category": "dataset",
"name": "header",
"function": "count",
"comparison": "NA",
"value": 1
},
{
"category": "dataset",
"name": "footer",
"function": "count",
"comparison": "NA",
"value": 0
},
{
"category": "format",
"name": "separator",
"function": "value",
"comparison": "NA",
"value": ","
},
{
"category": "format",
"name": "columns",
"function": "value",
"comparison": "NA",
"value": ","
},
{
"category": "dataset",
"name": "records",
"function": "count",
"comparison": "EQ",
"value": 15057
},
{
"category": "dataset",
"name": "columns",
"function": "count",
"comparison": "EQ",
"value": 1
}
]
},
{
"title": "BSBDirectoryJul21-304",
"uid": "4f57b82a-6b6a-4612-8d4e-67289b7795e1",
"documentName": "BSBDirectoryJul21-304.csv",
"issuedDate": "2021-10-23T18:37:58",
"modified": "2021-10-23T18:37:51",
"state": "active",
"author": "Meerkat@merebox.com",
"length": 1351065,
"hash": {
"algo": "SHA256",
"value": "5306b8baaafaf60600b645f3514dde3f654ac2b0bdbae96b0d0501b18488b1b5",
"signed": false
},
"description": "",
"url": "ftp://bsb.hostedftp.com/~auspaynetftp/BSB/BSBDirectoryJul21-304.csv",
"version": "",
"content-type": "text/csv",
"compression": null,
"encryption": null,
"attributes": [
{
"category": "dataset",
"name": "header",
"function": "count",
"comparison": "NA",
"value": 1
},
{
"category": "dataset",
"name": "footer",
"function": "count",
"comparison": "NA",
"value": 0
},
{
"category": "format",
"name": "separator",
"function": "value",
"comparison": "NA",
"value": ","
},
{
"category": "format",
"name": "columns",
"function": "value",
"comparison": "NA",
"value": ","
},
{
"category": "dataset",
"name": "records",
"function": "count",
"comparison": "EQ",
"value": 15044
},
{
"category": "dataset",
"name": "columns",
"function": "count",
"comparison": "EQ",
"value": 8
}
]
},
{
"title": "BSBDirectoryJul21-304",
"uid": "e8bbdfc4-ab46-465c-af97-39e4e4a3e36c",
"documentName": "BSBDirectoryJul21-304.txt",
"issuedDate": "2021-10-23T18:37:58",
"modified": "2021-10-23T18:37:53",
"state": "active",
"author": "Meerkat@merebox.com",
"length": 1700198,
"hash": {
"algo": "SHA256",
"value": "60d14b4ae28c8e91499c73e385504ea904134c0fa348af25a7baf58eb9eb8390",
"signed": false
},
"description": "",
"url": "ftp://bsb.hostedftp.com/~auspaynetftp/BSB/BSBDirectoryJul21-304.txt",
"version": "",
"content-type": "text/plain",
"compression": null,
"encryption": null,
"attributes": [
{
"category": "dataset",
"name": "header",
"function": "count",
"comparison": "NA",
"value": 1
},
{
"category": "dataset",
"name": "footer",
"function": "count",
"comparison": "NA",
"value": 0
},
{
"category": "format",
"name": "separator",
"function": "value",
"comparison": "NA",
"value": ","
},
{
"category": "format",
"name": "columns",
"function": "value",
"comparison": "NA",
"value": ","
},
{
"category": "dataset",
"name": "records",
"function": "count",
"comparison": "EQ",
"value": 15046
},
{
"category": "dataset",
"name": "columns",
"function": "count",
"comparison": "EQ",
"value": 1
}
]
},
{
"title": "BSBDirectorySep21-306",
"uid": "08d11601-54a6-4fb0-b694-e2d2c2a7eb32",
"documentName": "BSBDirectorySep21-306.csv",
"issuedDate": "2021-10-23T18:37:58",
"modified": "2021-10-23T18:37:55",
"state": "active",
"author": "Meerkat@merebox.com",
"length": 1351281,
"hash": {
"algo": "SHA256",
"value": "4f5c97799f6006247fad64f9a5acc425e79f31342cbbbdaaba2995af16ac56b4",
"signed": false
},
"description": "",
"url": "ftp://bsb.hostedftp.com/~auspaynetftp/BSB/BSBDirectorySep21-306.csv",
"version": "",
"content-type": "text/csv",
"compression": null,
"encryption": null,
"attributes": [
{
"category": "dataset",
"name": "header",
"function": "count",
"comparison": "NA",
"value": 1
},
{
"category": "dataset",
"name": "footer",
"function": "count",
"comparison": "NA",
"value": 0
},
{
"category": "format",
"name": "separator",
"function": "value",
"comparison": "NA",
"value": ","
},
{
"category": "format",
"name": "columns",
"function": "value",
"comparison": "NA",
"value": ","
},
{
"category": "dataset",
"name": "records",
"function": "count",
"comparison": "EQ",
"value": 15050
},
{
"category": "dataset",
"name": "columns",
"function": "count",
"comparison": "EQ",
"value": 8
}
]
},
{
"title": "BSBDirectorySep21-306",
"uid": "a4a01312-a254-4ba9-b9c7-294b1589a2e1",
"documentName": "BSBDirectorySep21-306.txt",
"issuedDate": "2021-10-23T18:37:58",
"modified": "2021-10-23T18:37:58",
"state": "active",
"author": "Meerkat@merebox.com",
"length": 1700876,
"hash": {
"algo": "SHA256",
"value": "dd830996c087b5b781a8d4d2bf7ec8da2b0de642a338ffbadea0d8d3834e2232",
"signed": false
},
"description": "",
"url": "ftp://bsb.hostedftp.com/~auspaynetftp/BSB/BSBDirectorySep21-306.txt",
"version": "",
"content-type": "text/plain",
"compression": null,
"encryption": null,
"attributes": [
{
"category": "dataset",
"name": "header",
"function": "count",
"comparison": "NA",
"value": 1
},
{
"category": "dataset",
"name": "footer",
"function": "count",
"comparison": "NA",
"value": 0
},
{
"category": "format",
"name": "separator",
"function": "value",
"comparison": "NA",
"value": ","
},
{
"category": "format",
"name": "columns",
"function": "value",
"comparison": "NA",
"value": ","
},
{
"category": "dataset",
"name": "records",
"function": "count",
"comparison": "EQ",
"value": 15052
},
{
"category": "dataset",
"name": "columns",
"function": "count",
"comparison": "EQ",
"value": 1
}
]
}
],
"custom": [
{
"extension": "software",
"softwareName": "MARTILQREFERENCE",
"author": "Meerkat@merebox.com",
"version": "0.0.1"
}
]
}

223
docs/samples/BSBDirectoryHttp.json 100644
View File

@ -0,0 +1,223 @@
{
"content-type": "application/vnd.martilq.json",
"title": "SampleFetchBSB",
"uid": "08e22a4d-5a11-4b19-9172-feb36601009e",
"description": "",
"modified": "2021-10-23",
"publisher": "meerkat",
"contactPoint": "",
"accessLevel": "Public",
"rights": "Publish",
"tags": ["BSB", "Australia"],
"license": "",
"state": "active",
"batch": 1.0,
"describedBy": "",
"landingPage": "",
"theme": "",
"resources": [
{
"title": "BSBDirectoryAug21-305",
"uid": "59f1c10e-a71b-45a4-9205-2b651cf04304",
"documentName": "BSBDirectoryAug21-305.csv",
"issuedDate": "2021-10-23T18:37:58",
"modified": "2021-10-23T18:37:46",
"state": "active",
"author": "Meerkat@merebox.com",
"length": 1352016,
"hash": {
"algo": "SHA256",
"value": "26e40760989ff8a314cda70628c38b66d0d565ea8ad677b81b83780e33984104",
"signed": false
},
"description": "",
"url": "http://apnedata.merebox.com.s3.ap-southeast-2.amazonaws.com/au/bsb/BSBDirectoryAug21-305.csv",
"version": "",
"content-type": "text/csv",
"compression": null,
"encryption": null,
"attributes": [
{
"category": "dataset",
"name": "header",
"function": "count",
"comparison": "NA",
"value": 1
},
{
"category": "dataset",
"name": "footer",
"function": "count",
"comparison": "NA",
"value": 0
},
{
"category": "format",
"name": "separator",
"function": "value",
"comparison": "NA",
"value": ","
},
{
"category": "format",
"name": "columns",
"function": "value",
"comparison": "NA",
"value": ","
},
{
"category": "dataset",
"name": "records",
"function": "count",
"comparison": "EQ",
"value": 15055
},
{
"category": "dataset",
"name": "columns",
"function": "count",
"comparison": "EQ",
"value": 8
}
]
},
{
"title": "BSBDirectoryJul21-304",
"uid": "4f57b82a-6b6a-4612-8d4e-67289b7795e1",
"documentName": "BSBDirectoryJul21-304.csv",
"issuedDate": "2021-10-23T18:37:58",
"modified": "2021-10-23T18:37:51",
"state": "active",
"author": "Meerkat@merebox.com",
"length": 1351065,
"hash": {
"algo": "SHA256",
"value": "5306b8baaafaf60600b645f3514dde3f654ac2b0bdbae96b0d0501b18488b1b5",
"signed": false
},
"description": "",
"url": "http://apnedata.merebox.com.s3.ap-southeast-2.amazonaws.com/au/bsb/BSBDirectoryJul21-304.csv",
"version": "",
"content-type": "text/csv",
"compression": null,
"encryption": null,
"attributes": [
{
"category": "dataset",
"name": "header",
"function": "count",
"comparison": "NA",
"value": 1
},
{
"category": "dataset",
"name": "footer",
"function": "count",
"comparison": "NA",
"value": 0
},
{
"category": "format",
"name": "separator",
"function": "value",
"comparison": "NA",
"value": ","
},
{
"category": "format",
"name": "columns",
"function": "value",
"comparison": "NA",
"value": ","
},
{
"category": "dataset",
"name": "records",
"function": "count",
"comparison": "EQ",
"value": 15044
},
{
"category": "dataset",
"name": "columns",
"function": "count",
"comparison": "EQ",
"value": 8
}
]
},
{
"title": "BSBDirectorySep21-306",
"uid": "08d11601-54a6-4fb0-b694-e2d2c2a7eb32",
"documentName": "BSBDirectorySep21-306.csv",
"issuedDate": "2021-10-23T18:37:58",
"modified": "2021-10-23T18:37:55",
"state": "active",
"author": "Meerkat@merebox.com",
"length": 1351281,
"hash": {
"algo": "SHA256",
"value": "4f5c97799f6006247fad64f9a5acc425e79f31342cbbbdaaba2995af16ac56b4",
"signed": false
},
"description": "",
"url": "http://apnedata.merebox.com.s3.ap-southeast-2.amazonaws.com/au/bsb/BSBDirectorySep21-306.csv",
"version": "",
"content-type": "text/csv",
"compression": null,
"encryption": null,
"attributes": [
{
"category": "dataset",
"name": "header",
"function": "count",
"comparison": "NA",
"value": 1
},
{
"category": "dataset",
"name": "footer",
"function": "count",
"comparison": "NA",
"value": 0
},
{
"category": "format",
"name": "separator",
"function": "value",
"comparison": "NA",
"value": ","
},
{
"category": "format",
"name": "columns",
"function": "value",
"comparison": "NA",
"value": ","
},
{
"category": "dataset",
"name": "records",
"function": "count",
"comparison": "EQ",
"value": 15050
},
{
"category": "dataset",
"name": "columns",
"function": "count",
"comparison": "EQ",
"value": 8
}
]
}
],
"custom": [
{
"extension": "software",
"softwareName": "MARTILQREFERENCE",
"author": "Meerkat@merebox.com",
"version": "0.0.1"
}
]
}

4
docs/samples/powershell/Invoke-SampleGenerateBsb.ps1
View File

@ -152,7 +152,7 @@ foreach($file in Get-ChildItem $localDirectory)
}
}
$oResource = New-MartiResource -SourcePath $zipFile -UrlPath $localDirectory -LogPath ".\test\Logs" -ExtendAttributes
Set-AttributeValueString -Attributes $oResource.attributes -Key "compression" -Category "format" -Function "algorithm" -Value "WINZIP"
Set-AttributeValueString -Attributes $oResource.attributes -Key "compression" -Category "format" -Function "algo" -Value "WINZIP"
$oMarti.resources += $oResource
$fileJson = Join-Path -Path $localDirectory -ChildPath "MartiBSBZip.mti"
@ -224,7 +224,7 @@ Compress-7Zip -Path $noticeFile -ArchiveFileName $zipFile -Append -Password $sec
$oResource = New-MartiResource -SourcePath $zipFile -UrlPath $localDirectory -LogPath ".\test\Logs" -ExtendAttributes
$oResource.compression = "7ZIP"
$oResource.encryption = New-Encryption -Algorithm "Passphrase" -Value $secret
Set-AttributeValueString -Attributes $oResource.attributes -Key "compression" -Category "format" -Function "algorithm" -Value "7ZIP"
Set-AttributeValueString -Attributes $oResource.attributes -Key "compression" -Category "format" -Function "algo" -Value "7ZIP"
$oMarti.resources += $oResource
$fileJson = Join-Path -Path $localDirectory -ChildPath "MartiBSBSecure.mti"

2
docs/samples/powershell/Invoke-SampleGenerateBsbSecure.ps1
View File

@ -161,7 +161,7 @@ $oResource.encryption = New-Encryption -Algorithm "PKI" -Value $($encryptedSecre
Write-Debug "Secret: $secret"
Set-AttributeValueString -Attributes $oResource.attributes -Key "compression" -Category "format" -Function "algorithm" -Value "7ZIP"
Set-AttributeValueString -Attributes $oResource.attributes -Key "compression" -Category "format" -Function "algo" -Value "7ZIP"
$oMarti.resources += $oResource
$fileJson = Join-Path -Path $localDirectory -ChildPath "MartiBSB7ZipPKI.mti"

56
docs/samples/python/SampleFetchFtpBsb.py 100644
View File

@ -0,0 +1,56 @@
import os
import sys
import csv
sys.path.insert(0, "../../../source/python/client")
from martiLQ import *
test_directory = "./test/fetch_ftp"
os.environ["MARTILQ_LOGPATH"] = os.path.join(test_directory, "logs")
if not os.path.exists("./test"):
os.mkdir("./test")
if not os.path.exists(test_directory):
os.mkdir(test_directory)
print("Creating martiLQ definition")
mlq = martiLQ()
mlq.LoadConfig()
print("Loading definition json")
mlq.Load("../BSBDirectoryFtp.json")
print("Fetching files")
fetched_files, fetch_error = mlq.Fetch(test_directory)
if len(fetched_files) < 0:
raise Exception("No resource files fetched")
else:
print("Fetched {} files".format(len(fetched_files)))
if len(fetch_error) > 0:
raise Exception("Some resources not fetched")
print("Generate the self value, overriding existing")
oMarti = mlq.NewMartiDefinition()
for full_fileName in fetched_files:
if os.path.isfile(full_fileName):
oResource = mlq.NewMartiLQResource(full_fileName, "", False, True)
oMarti["resources"].append(oResource)
print("Perform validation test")
lqresults, testError = mlq.TestMartiDefinition("../BSBDirectoryFtp.json")
testfile = open("./test/LoadQualityTest_Ftp.csv", "w+", newline ="")
with testfile:
lqwriter = csv.writer(testfile)
lqwriter.writerows(lqresults)
if testError > 0:
print("MISMATCH DETECTED")
else:
print("RECONCILED")
mlq.CloseLog()
print("Test completed: SampleFetchFtpBsb.py")

56
docs/samples/python/SampleFetchHttpBsb.py 100644
View File

@ -0,0 +1,56 @@
import os
import sys
import csv
sys.path.insert(0, "../../../source/python/client")
from martiLQ import *
test_directory = "./test/fetch_http"
os.environ["MARTILQ_LOGPATH"] = os.path.join(test_directory, "logs")
if not os.path.exists("./test"):
os.mkdir("./test")
if not os.path.exists(test_directory):
os.mkdir(test_directory)
print("Creating martiLQ definition")
mlq = martiLQ()
mlq.LoadConfig()
print("Loading definition json")
mlq.Load("../BSBDirectoryHttp.json")
print("Fetching files")
fetched_files, fetch_error = mlq.Fetch(test_directory)
if len(fetched_files) < 0:
raise Exception("No resource files fetched")
else:
print("Fetched {} files".format(len(fetched_files)))
if len(fetch_error) > 0:
raise Exception("Some resources not fetched")
print("Generate the self value, overriding existing")
oMarti = mlq.NewMartiDefinition()
for full_fileName in fetched_files:
if os.path.isfile(full_fileName):
oResource = mlq.NewMartiLQResource(full_fileName, "", False, True)
oMarti["resources"].append(oResource)
print("Perform validation test")
lqresults, testError = mlq.TestMartiDefinition("../BSBDirectoryHttp.json")
testfile = open("./test/LoadQualityTest_Http.csv", "w+", newline ="")
with testfile:
lqwriter = csv.writer(testfile)
lqwriter.writerows(lqresults)
if testError > 0:
print("MISMATCH DETECTED")
else:
print("RECONCILED")
mlq.CloseLog()
print("Test completed: SampleFetchHttpBsb.py")

57
docs/samples/python/SampleGenerateBsb.py
View File

@ -4,11 +4,15 @@ import os
import json
import sys
import csv
import zipfile
sys.path.insert(0, "../../../source/python/client")
#from source.python.client.martiLQ import martiLQ
from martiLQ import *
ftpFetch = True
os.environ["MARTILQ_LOGPATH"] = "./test/logs"
def ftpList(host, path):
files = []
@ -58,7 +62,8 @@ for file_name in files:
if file_name.endswith(".csv") | file_name.endswith(".txt"):
file_remote = remote_dir + file_name
file_local = "./test/" + file_name
# ftpPull(remote_host, file_remote, file_local)
if ftpFetch:
ftpPull(remote_host, file_remote, file_local)
print("Creating martiLQ definition")
mlq = martiLQ()
@ -67,16 +72,56 @@ oMarti = mlq.NewMartiDefinition()
for file_name in files:
if file_name.startswith("BSBDirectory"):
if file_name.endswith(".csv") | file_name.endswith(".txt"):
oResource = mlq.NewMartiResource(os.path.join("./test/", file_name), "", False, True, "./test/logs")
oResource = mlq.NewMartiLQResource(os.path.join("./test/", file_name), "", False, True)
oMarti["resources"].append(oResource)
mlq.CloseLog()
print("Save martiLQ definition")
jd = json.dumps(oMarti, indent=5)
print("Save martiLQ definition")
jsonFile = open("./test/BSBDirectoryPlain.mti", "w")
jsonFile.write(jd)
jsonFile.write(json.dumps(oMarti, indent=5))
jsonFile.close()
print("Base sample mti written: BSBDirectoryPlain.mti")
print("Creating martiLQ ZIP file")
zipFileName = "BSBDirectory.zip"
fileZipCount = 0
mlq = martiLQ()
oMarti = mlq.NewMartiDefinition()
with zipfile.ZipFile("./test/" + zipFileName, "w", compression=zipfile.ZIP_DEFLATED) as zipObj:
for file_name in files:
if file_name.startswith("BSBDirectory"):
if file_name.endswith(".csv") | file_name.endswith(".txt"):
file_remote = remote_dir + file_name
file_local = "./test/" + file_name
if ftpFetch:
ftpPull(remote_host, file_remote, file_local)
zipObj.write(file_local, file_name)
fileZipCount = fileZipCount + 1
oResource = mlq.NewMartiLQResource(os.path.join("./test/", file_name), "", False, True)
oResource["url"] = "@"+zipFileName + "/" + file_name
oMarti["resources"].append(oResource)
oResource = mlq.NewMartiLQResource(os.path.join("./test/", zipFileName), "", False, True)
oResource["url"] = "./test/" + zipFileName
mlq.SetAttributeValueString(Attributes=oResource["attributes"], Key="compression", Category="format", Function="algo", Value="WINZIP")
mlq.SetAttributeValueNumber(Attributes=oResource["attributes"], Key="files", Category="dataset", Function="count", Value=fileZipCount)
oMarti["resources"].append(oResource)
mlq.CloseLog()
print("Save martiLQ ZIP definition")
jsonFile = open("./test/MartiLQ_BSBZip.mti", "w")
jsonFile.write(json.dumps(oMarti, indent=5))
jsonFile.close()
print("ZIP sample mti written: MartiLQ_BSBZip.mti")
print("Sample completed: SampleGenerateBsb.py")
lqresults, testError = mlq.TestMartiDefinition("./test/BSBDirectoryPlain.mti")

92
docs/samples/python/SampleGenerateBsbSecure.py 100644
View File

@ -0,0 +1,92 @@
import ftplib
import os
import json
import sys
import csv
sys.path.insert(0, "../../../source/python/client")
from martiLQ import *
def ftpList(host, path):
files = []
with ftplib.FTP(host) as ftp:
try:
ftp.login()
ftp.cwd(path)
files = ftp.nlst()
except ftplib.all_errors as e:
print('FTP error:', e)
return files
def ftpPull(host, file_remote, file_local):
with ftplib.FTP(host) as ftp:
try:
ftp.login()
with open(file_local, 'wb') as fl:
res = ftp.retrbinary(f"RETR {file_remote}", fl.write)
if not res.startswith('226 Transfer complete'):
print('Download failed')
if os.path.isfile(file_local):
os.remove(file_local)
except ftplib.all_errors as e:
print('FTP error:', e)
if os.path.isfile(file_local):
os.remove(file_local)
remote_host = "bsb.hostedftp.com"
remote_dir = "/~auspaynetftp/BSB/"
print("Fetch sample file list")
files = ftpList(remote_host, remote_dir)
if not os.path.exists("./test"):
os.mkdir("./test")
print("Fetch sample files")
for file_name in files:
if file_name.startswith("BSBDirectory"):
if file_name.endswith(".csv") | file_name.endswith(".txt"):
file_remote = remote_dir + file_name
file_local = "./test/" + file_name
ftpPull(remote_host, file_remote, file_local)
print("Creating martiLQ definition")
mlq = martiLQ()
oMarti = mlq.NewMartiDefinition()
for file_name in files:
if file_name.startswith("BSBDirectory"):
if file_name.endswith(".csv") | file_name.endswith(".txt"):
oResource = mlq.NewMartiLQResource(os.path.join("./test/", file_name), "", False, True, "./test/logs")
oMarti["resources"].append(oResource)
mlq.CloseLog()
print("Save martiLQ definition")
jd = json.dumps(oMarti, indent=5)
jsonFile = open("./test/BSBDirectoryPlain.mti", "w")
jsonFile.write(jd)
jsonFile.close()
print("Sample completed: SampleGenerateBsb.py")
lqresults, testError = mlq.TestMartiDefinition("./test/BSBDirectoryPlain.mti")
testfile = open("./test/LoadQualityTest01.csv", "w+", newline ="")
with testfile:
lqwriter = csv.writer(testfile)
lqwriter.writerows(lqresults)
if testError:
print("MISMATCH DETECTED")
print("Test completed: SampleGenerateBsb.py")

11
source/java/README.md
View File

@ -1,3 +1,12 @@
# Place maker
# Java code for martiLQ
The ``client`` folder contains Java library for creating
**martiLQ** definition and comparison for the Load Quality.
See the Java samples directory on how to use the code.
__TO COME__
Publish of Java library to ``Maven`` is yet to occur
This will occur after merge of code into ``main`` branch

0
source/powershell/Compare-MartiResource.ps1 → source/powershell/Compare-MartiLQResource.ps1
View File

6
source/powershell/MartiLQItem.ps1
View File

@ -144,7 +144,7 @@ Param(
Get-ChildItem $SourceFolder -Filter $Filter -Recurse:$Recurse -Force| Where-Object {!$_.PSIsContainer} | ForEach-Object {
$oResource = New-MartiResource -SourcePath $_.FullName -UrlPath $remoteDirectory -LogPath $LogPath -ExtendAttributes:$ExtendAttributes -ExcludeHash:$ExcludeHash
$oResource = New-MartiResource -SourcePath $_.FullName -UrlPath $UrlPath -LogPath $LogPath -ExtendAttributes:$ExtendAttributes -ExcludeHash:$ExcludeHash
if ($null -ne $UrlPath -and $UrlPath -ne "") {
$postfixName = $_.FullName.Replace($SourceFullName, "")
@ -289,7 +289,7 @@ function New-DefaultZipAttributes {
$oAttribute = [PSCustomObject]@{
category = "format"
name = "compression"
function = "algorithm"
function = "algo"
comparison = "NA"
value = $CompressionType
}
@ -298,7 +298,7 @@ function New-DefaultZipAttributes {
$oAttribute = [PSCustomObject]@{
category = "format"
name = "encryption"
function = "algorithm"
function = "algo"
comparison = "NA"
value = $Encryption
}

12
source/powershell/README.md
View File

@ -1,3 +1,13 @@
# Place maker
# PowerShell code for martiLQ
This folder contains PowerShell module for creating
**martiLQ** definition and comparison for the Load Quality.
See the PowerShell samples directory on how to use the code.
__TO COME__
Publish of PowerShell code as ``Install-Module martiLQ`` is yet to occur
This will occur after merge of code into ``main`` branch

11
source/python/README.md
View File

@ -1,3 +1,12 @@
# Place maker
# Python code for martiLQ
The ``client`` folder contains Python module for creating
**martiLQ** definition and comparison for the Load Quality.
See the Python samples directory on how to use the code.
__TO COME__
Publish of Python code as ``pip install martiLQ`` is yet to occur
This will occur after merge of code into ``main`` branch

455
source/python/client/martiLQ.py
View File

@ -8,47 +8,172 @@ import json
import datetime
import getpass
import hashlib
import glob
from configparser import ConfigParser
import requests
import mimetypes
class martiLQ:
gLogPathName = ""
gSoftwareVersion = "0.0.1"
gdefault_metaFile = "##marti##.mti"
_SoftwareVersion = "0.0.1"
_default_metaFile = "##marti##.mti"
gMartiErrorId = ""
gLogOpen = False
_oSoftware = {
"extension": "software",
"softwareName": "MARTILQREFERENCE",
"author": "Meerkat@merebox.com",
"version": "0.0.1"
}
oMartiDefinition = None
def __init__(self):
self.gLogOpen = False
_MartiErrorId = ""
_LogOpen = False
def Get(self):
return self.oMartiDefinition
_oMartiDefinition = None
_oConfiguration = None
def Close(self):
if self.gLogOpen:
self.CloseLog()
self.gLogOpen = False
def GetSoftwareName(self):
return "MARTILQREFERENCE"
def __init__(self):
self._LogOpen = False
_oSoftware = {
"extension": "software",
"softwareName": self.GetSoftwareName(),
"author": "Meerkat@merebox.com",
"version": self._SoftwareVersion
}
self._oConfiguration = {
"softwareName": self.GetSoftwareName(),
"author": "Meerkat@merebox.com",
"version": self._SoftwareVersion,
"logPath": None,
"state": "active",
"accessLevel": "Confidential",
"rights": "Restricted",
"hashAlgorithm": "SHA256",
"signKey_File": None,
"signKey_Password": None,
"loaded": False
}
def LoadConfig(self, ConfigPath=None):
config_object = ConfigParser()
if not ConfigPath is None:
if os.path.exists(ConfigPath):
config_object.read(ConfigPath)
else:
self.WriteLog("Configuration path '{}' does not exist".format(ConfigPath))
raise Exception("Configuration path '{}' does not exist".format(ConfigPath))
else:
# Look in default location and name
home = os.path.expanduser('~')
if os.path.exists(os.path.join(home, ".martilq/martilq.ini")):
ConfigPath = os.path.join(home, ".martilq/martilq.ini")
if os.path.exists("martilq.ini"):
ConfigPath = "martilq.ini"
if not ConfigPath is None:
self.WriteLog("Usig configuration path '{}'".format(ConfigPath))
config_object.read(ConfigPath)
if config_object.has_section("Resources"):
items = config_object["Resource"]
if not items is None:
config_attr = ["accessLevel", "rights", "state"]
for x in config_attr:
if not items[x] is None and not items[x] == "":
self._oConfiguration[x] = items[x]
if config_object.has_section("Hash"):
items = config_object["Hash"]
if not items is None:
config_attr = ["hashAlgorithm", "signKey_File", "signKey_Password"]
for x in config_attr:
if not items[x] is None and not items[x] == "":
self._oConfiguration[x] = items[x]
# Now check environmental values
self._oConfiguration["signKey_File"] = os.getenv("MARTILQ_SIGNKEY_FILE", self._oConfiguration["signKey_File"])
self._oConfiguration["signKey_Password"] = os.getenv("MARTILQ_SIGNKEY_PASSWORD", self._oConfiguration["signKey_Password"])
self._oConfiguration["logPath"] = os.getenv("MARTILQ_LOGPATH", self._oConfiguration["logPath"])
self.WriteLog("Configuration load processed")
def Set(self, MartiLQ):
self._oMartiDefinition = MartiLQ
def Get(self):
return self._oMartiDefinition
def Save(self, JsonPath):
jsonFile = open(JsonPath, "w")
jsonFile.write(json.dumps(self._oMartiDefinition, indent=5))
jsonFile.close()
def Load(self, JsonPath):
self._MartiErrorId = ""
self.OpenLog()
self.WriteLog("Function 'Load' parameters follow")
self.WriteLog("Parameter: SourcePath Value: {}".format(JsonPath))
self.WriteLog("")
if os.path.exists(JsonPath):
self.WriteLog("Overwriting existing definition")
else:
if not os.path.exists(os.path.dirname(JsonPath)):
self.WriteLog("Parent folder does not exist")
raise Exception("Parent folder '{}' does not exist".format(os.path.dirname(JsonPath)))
if not self._oMartiDefinition is None:
self.WriteLog("Existing definition overwritten")
jsonFile = open(JsonPath, "r")
self._oMartiDefinition = json.load(jsonFile)
jsonFile.close()
def SetConfig(self, Key=None, Value=None):
if not Key is None:
self._oConfiguration[Key] = Value
def GetConfig(self, Key=None):
if not Key is None:
return self._oConfiguration[Key]
else:
return None
def Close(self):
if self._LogOpen:
self.CloseLog()
self._LogOpen = False
def GetLogName(self):
today = datetime.datetime.today()
dateToday = today.strftime("%Y-%m-%d")
if None == self.gLogPathName or self.gLogPathName == "":
if None == self._oConfiguration["logPath"] or self._oConfiguration["logPath"] == "":
return None
if not os.path.exists(self.gLogPathName):
os.mkdir(self.gLogPathName)
if not os.path.exists(self._oConfiguration["logPath"]):
os.mkdir(self._oConfiguration["logPath"])
logName = self.GetSoftwareName() + "_" + dateToday + ".log"
return os.path.join(self.gLogPathName, logName)
return os.path.join(self._oConfiguration["logPath"], logName)
def WriteLog(self, LogEntry):
@ -75,55 +200,48 @@ class martiLQ:
def OpenLog(self):
if not self.gLogOpen:
if not self._LogOpen:
today = datetime.datetime.today()
dateToday = today.strftime("%Y-%m-%d")
self.WriteLog("***********************************************************************************")
self.WriteLog("* Start of processing: {}".format(dateToday))
self.WriteLog("***********************************************************************************")
self.gLogOpen = True
self._LogOpen = True
def CloseLog(self):
if self.gLogOpen:
if self._LogOpen:
today = datetime.datetime.today()
dateToday = today.strftime("%Y-%m-%d")
self.WriteLog("***********************************************************************************")
self.WriteLog("* End of processing: {}".format(dateToday))
self.WriteLog("***********************************************************************************")
self.gLogOpen = False
self._LogOpen = False
def NewMartiDefinition(self):
today = datetime.datetime.today()
dateToday = today.strftime("%Y-%m-%d")
oSoftware = {
"extension": "software",
"softwareName": self.GetSoftwareName(),
"author": "Meerkat@merebox.com",
"version": self.gSoftwareVersion
}
publisher = getpass.getuser()
lcustom = []
lcustom.append(oSoftware)
lcustom.append(self._oSoftware)
lresource = []
self.oMartiDefinition = {
self._oMartiDefinition = {
"content-type": "application/vnd.martilq.json",
"title": "",
"uid": str(uuid.uuid4()),
"resources": lresource,
"description": "",
"modified": dateToday,
"tags": ["document", self.GetSoftwareName()],
"publisher": publisher,
"contactPoint": "",
"accessLevel": "Confidential",
"rights": "Restricted",
"accessLevel": self.GetConfig("accessLevel"),
"rights": self.GetConfig("rights"),
"tags": [],
"license": "",
"state": "active",
"batch": 1.0,
@ -131,23 +249,54 @@ class martiLQ:
"landingPage": "",
"theme": "",
"resources": lresource,
"custom": lcustom
}
return self.oMartiDefinition
return self._oMartiDefinition
def NewMartiChildItem(self, SourceFolder, UrlPath=None, Recurse=True, ExtendAttributes=True, ExcludeHash=False, Filter ="*"):
def NewMartiResource(self, SourcePath, UrlPath, ExcludeHash, ExtendAttributes, LogPath):
SourceFullName = os.path.abspath(SourceFolder)
for fullName in glob.iglob(SourceFullName, recursive=Recurse):
if os.path.isfile(fullName):
oResource = self.NewMartiLQResource(SourcePath=fullName, UrlPath=UrlPath, ExtendAttributes=ExtendAttributes, ExcludeHash=ExcludeHash)
self._oMartiDefinition["resources"].append(oResource)
def GetContentType(self, SourcePath):
ext = None
# Some overrides
match = str(os.path.splitext(SourcePath)[1][1:]).lower()
if (ext is None or ext == "") and match == "csv":
ext = "text/csv"
if ext is None or ext == "":
ext, _ = mimetypes.guess_type(SourcePath, strict=True)
if ext is None or ext == "":
match = str(os.path.splitext(SourcePath)[1][1:]).lower()
if match == "md":
ext = "application/markdown"
if ext is None or ext == "":
ext = "application/vnd.unknown." + os.path.splitext(SourcePath)[1][1:]
return ext
def NewMartiLQResource(self, SourcePath, UrlPath, ExcludeHash, ExtendAttributes):
today = datetime.datetime.today()
dateToday = today.strftime("%Y-%m-%d")
dateToday = today.strftime("%Y-%m-%dT%H:%M:%S")
self.gMartiErrorId = ""
self.gLogPathName = LogPath
self._MartiErrorId = ""
self.OpenLog()
self.WriteLog("Function 'NewMartiResource' parameters follow")
self.WriteLog("Function 'NewMartiLQResource' parameters follow")
self.WriteLog("Parameter: SourcePath Value: {}".format(SourcePath))
self.WriteLog("Parameter: UrlPath Value: {}".format(UrlPath))
self.WriteLog("Parameter: ExcludeHash Value: {}".format(ExcludeHash))
@ -157,20 +306,20 @@ class martiLQ:
item = os.path.basename(SourcePath)
self.WriteLog("Define file {}".format(SourcePath))
HashAlgorithm = "SHA256"
HashAlgorithm = self.GetConfig("hashAlgorithm")
try:
mtime = os.path.getmtime(SourcePath)
except OSError:
mtime = 0
last_modified_date = datetime.datetime.fromtimestamp(mtime).strftime("%Y-%m-%d %H:%M:%S")
last_modified_date = datetime.datetime.fromtimestamp(mtime).strftime("%Y-%m-%dT%H:%M:%S")
if ExcludeHash:
hash = None
else:
hash = self.NewMartiLQHash(Algorithm=HashAlgorithm, FilePath=SourcePath, Value="")
hash = self.NewMartiLQHash(Algorithm=HashAlgorithm, FilePath=SourcePath, Value="", Sign=self.GetConfig("signKey_File"))
lattribute = self.SetMartiResourceAttributes(SourcePath, os.path.splitext(SourcePath)[1][1:], ExtendAttributes)
lattribute = self.SetMartiLQResourceAttributes(SourcePath, str(os.path.splitext(SourcePath)[1][1:]).lower(), ExtendAttributes)
oResource = {
"title": item.replace(os.path.splitext(SourcePath)[1], ""),
@ -178,15 +327,15 @@ class martiLQ:
"documentName": item,
"issuedDate": dateToday,
"modified": last_modified_date,
"state": "active",
"author": "",
"state": self.GetConfig("state"),
"author": self.GetConfig("author"),
"length": os.path.getsize(SourcePath),
"hash": hash,
"description": "",
"url": "",
"version": self.gSoftwareVersion,
"format": os.path.splitext(SourcePath)[1][1:],
"version": "",
"content-type": self.GetContentType(SourcePath),
"compression": None,
"encryption": None,
@ -194,7 +343,7 @@ class martiLQ:
}
if None != UrlPath and UrlPath != "":
if UrlPath[UrlPath.Length-1] == "/" or UrlPath[UrlPath.Length-1] == "\\":
if UrlPath[len(UrlPath)-1] == "/" or UrlPath[len(UrlPath)-1] == "\\":
oResource["url"] = UrlPath.replace("\\", "/") + item
else:
oResource["url"] = UrlPath.replace("\\", "/") + "/" + item
@ -202,18 +351,27 @@ class martiLQ:
self.WriteLog("Complete file {}".format(SourcePath))
else:
self.gMartiErrorId = "MRI2001"
self._MartiErrorId = "MRI2001"
message = "Document '{}' not found or is a folder".format(SourcePath)
self.WriteLog(message + " " + self.gMartiErrorId)
self.WriteLog(message + " " + self._MartiErrorId)
raise Exception(message)
return oResource
def NewMartiLQHash(self, Algorithm, FilePath, Value=""):
def NewMartiLQHash(self, Algorithm, FilePath, Value="", Sign=""):
try:
signed = False
if Value == "" and FilePath != "":
if not Sign is None and not Sign == "" and not os.path.exists(Sign):
self.WriteLog("Sign file '{}' not found".format(Sign))
Sign = ""
if Sign is None or Sign == "":
if Algorithm == "SHA256":
sha_hash = hashlib.sha256()
with open(FilePath,"rb") as fh:
@ -233,11 +391,36 @@ class martiLQ:
sha_hash.update(byte_block)
Value = sha_hash.hexdigest()
else:
import OpenSSL
from OpenSSL import crypto
import base64
private_key_file = open(Sign, "r")
privkey = private_key_file.read()
private_key_file.close()
password = self.GetConfig("sigenKey_Password")
if privkey.startswith('-----BEGIN '):
pkey = crypto.load_privatekey(crypto.FILETYPE_PEM, privkey, password.encode('utf-8'))
else:
pkey = crypto.load_pkcs12(privkey, password).get_privatekey()
with open(FilePath,"rb") as fh:
sign = OpenSSL.crypto.sign(pkey, fh.read(), Algorithm)
Value = (base64.b64encode(sign)).decode('utf-8')
signed = True
oHash = {
"algo": Algorithm,
"value": Value
"value": Value,
"signed": signed
}
except Exception as e:
self.WriteLog("Hash error for file {}: {}".format(FilePath, str(e)))
raise e
return oHash
@ -251,10 +434,6 @@ class martiLQ:
return oEncryption
def SetMartiAttribute(self, Attributes, ACategory, AName, AFunction, Comparison, Value):
matched = False
@ -396,7 +575,7 @@ class martiLQ:
oAttribute = {
"category": "format",
"name": "compression",
"function": "algorithm",
"function": "algo",
"comparison": "NA",
"value": CompressionType
}
@ -405,7 +584,7 @@ class martiLQ:
oAttribute = {
"category": "format",
"name": "encryption",
"function": "algorithm",
"function": "algo",
"comparison": "NA",
"value": Encryption
}
@ -423,7 +602,7 @@ class martiLQ:
return lattribute
def SetAttributeValueString(self, Attributes,Category,Key,Function,Value,Comparison = "EQ"):
def SetAttributeValueString(self, Attributes, Category, Key, Function, Value, Comparison="EQ"):
for item in Attributes:
@ -447,7 +626,7 @@ class martiLQ:
return
def SetAttributeValueNumber(self, Attributes,Category,Key,Function,Value,Comparison = "EQ"):
def SetAttributeValueNumber(self, Attributes, Category, Key, Function, Value, Comparison = "EQ"):
for item in Attributes:
@ -472,7 +651,7 @@ class martiLQ:
return
def SetMartiResourceAttributes(self, PathFile, FileType, ExtendedAttributes):
def SetMartiLQResourceAttributes(self, PathFile, FileType, ExtendedAttributes):
lattribute = None
@ -526,10 +705,12 @@ class martiLQ:
if FileType == "zip":
lattribute = self.NewDefaultZipAttributes("ZIP")
if ExtendedAttributes:
self.SetAttributeValueNumber(lattribute, "dataset", "files", "count", -1)
self.SetAttributeValueNumber(lattribute, "dataset", "files", "count", 0, Comparison="NA")
if FileType == "7z":
lattribute = self.NewDefaultZipAttributes("7Z")
if ExtendedAttributes:
self.SetAttributeValueNumber(lattribute, "dataset", "files", "count", 0, Comparison="NA")
if lattribute == None:
lattribute = []
@ -537,6 +718,81 @@ class martiLQ:
return lattribute
def FtpPull(self, host, file_remote, file_local):
with ftplib.FTP(host) as ftp:
try:
ftp.login()
with open(file_local, 'wb') as fl:
res = ftp.retrbinary(f"RETR {file_remote}", fl.write)
if not res.startswith('226 Transfer complete'):
print('Download failed')
if os.path.isfile(file_local):
os.remove(file_local)
except ftplib.all_errors as e:
print('FTP error:', e)
if os.path.isfile(file_local):
os.remove(file_local)
def Fetch(self, TargetPath):
if TargetPath is None or TargetPath == "":
self.WriteLog("Target path is missing from fetch")
raise Exception("Target path is missing from fetch")
if self._oMartiDefinition is None:
self.WriteLog("No defintion loaded")
raise Exception("No defintion loaded")
if len(self._oMartiDefinition["resources"]) < 1:
self.WriteLog("No resources in defintion")
raise Exception("No resources in defintion")
fetched_files = []
fetch_error = []
for resource in self._oMartiDefinition["resources"]:
if not resource["url"] is None and not resource["url"] == "":
method = str(resource["url"].split(":", 2)[0]).lower()
#print("Method of fetch {} for {}".format(method, resource["url"]))
matched = False
if method == "ftp":
matched = True
parts = resource["url"].split("/", 3)
host = parts[2]
file_remote = parts[3]
self.FtpPull(host, file_remote, os.path.join(TargetPath, resource["documentName"]))
fetched_files.append(os.path.join(TargetPath, resource["documentName"]))
if method == "http" or method == "https":
matched = True
response = requests.get(resource["url"])
if not response.status_code == 200:
self.WriteLog("HTP fetch failed with code {} for '{}'".format(response.status_code, resource["url"]))
print("HTP fetch failed with code {} for '{}'".format(response.status_code, resource["url"]))
fetch_error.append(resource["url"])
else:
with open(os.path.join(TargetPath, resource["documentName"]),'wb') as fh:
fh.write(response.content)
fetched_files.append(os.path.join(TargetPath, resource["documentName"]))
if method == "file":
pass
if not matched:
fetch_error.append(resource["documentName"])
else:
fetch_error.append(resource["documentName"])
return fetched_files, fetch_error
def TestAttributeDefinition(self, oTestResults, documentName, localR, remoteR):
errorCount = 0
@ -564,57 +820,88 @@ class martiLQ:
return errorCount
def TestMartiDefinition(self, LQSourcePath, oMarti = None, LogPath =""):
def TestMartiDefinition(self, SourcePath, Sign=None):
self.gMartiErrorId = ""
self.gLogPathName = LogPath
self._MartiErrorId = ""
self.OpenLog()
self.WriteLog("Function 'TestMartiDefinition' parameters follow")
self.WriteLog("Parameter: oMarti Value: {}".format(oMarti))
self.WriteLog("Parameter: LQSourcePath Value: {}".format(LQSourcePath))
self.WriteLog("Parameter: SourcePath Value: {}".format(SourcePath))
self.WriteLog("")
if oMarti is None:
oMarti = self.oMartiDefinition
if oMarti is None:
if self._oMartiDefinition is None:
pass
if not os.path.exists(LQSourcePath):
if not os.path.exists(SourcePath):
pass
jsonFile = open(LQSourcePath, "r")
jsonFile = open(SourcePath, "r")
lq = json.load(jsonFile)
jsonFile.close()
testError = False
testError = 0
oTestResults = []
otest = ["ResourceName", "Level", "Metric", "Matches", "LocalCalculation", "SuppliedValue" ]
oTestResults.append(otest)
otest = ["", "Batch", "Resource count", (len(oMarti["resources"]) == len(lq["resources"])), len(oMarti["resources"]), len(lq["resources"]) ]
otest = ["", "Batch", "Resource count", (len(self._oMartiDefinition["resources"]) == len(lq["resources"])), len(self._oMartiDefinition["resources"]), len(lq["resources"]) ]
oTestResults.append(otest)
for resource in oMarti["resources"]:
for resource in self._oMartiDefinition["resources"]:
for retarget in lq["resources"]:
if resource["documentName"] == retarget["documentName"]:
testError = testError or resource["hash"]["value"] != retarget["hash"]["value"]
if retarget["hash"]["signed"]:
# Need to verify the hash
if Sign is None:
Sign = self.GetConfig("signKey_file")
if Sign is None:
self.WriteLog("No Sign Key specified so Hash check cannot be performed for signed content")
else:
try:
import OpenSSL
from OpenSSL import crypto
import base64
except ImportError:
self.WriteLog("Import error in signed verification")
pub_key_file = open(Sign, "r")
pubkey = pub_key_file.read()
pub_key_file.close()
x509 = crypto.X509()
x509.set_pubkey(pubkey)
try:
crypto.verify(x509, retarget["hash"]["value"], resource["hash"]["value"], retarget["hash"]["algo"])
otest = [resource["documentName"], "Resource", "Hash",False, resource["hash"]["value"], retarget["hash"]["value"] ]
except:
testError = testError + 1
self.WriteLog("Error in verification for {}".format(resource["documentName"]))
otest = [resource["documentName"], "Resource", "Hash", True, resource["hash"]["value"], retarget["hash"]["value"] ]
oTestResults.append(otest)
pass
else:
if not resource["hash"]["value"] == retarget["hash"]["value"]:
testError = testError + 1
otest = [resource["documentName"], "Resource", "Hash", (resource["hash"]["value"] == retarget["hash"]["value"]), resource["hash"]["value"], retarget["hash"]["value"] ]
oTestResults.append(otest)
testError = testError or resource["length"] != retarget["length"]
if not resource["length"] == retarget["length"]:
testError = testError + 1
otest = [resource["documentName"], "Resource", "Length", (resource["length"] == retarget["length"]), resource["length"], retarget["length"] ]
oTestResults.append(otest)
errorAttrCount = self.TestAttributeDefinition(oTestResults, resource["documentName"], resource["attributes"], retarget["attributes"])
if errorAttrCount > 0:
testError = True
testError = testError + errorAttrCount
break
self.WriteLog("TestMartiDefinition function completed with {} errors".format(testError))
return oTestResults, testError

44
test/powershell/test_Marti.ps1
View File

@ -1,44 +0,0 @@
. .\source\powershell\MartiLQ.ps1
. .\source\powershell\Compress-MartiLQ.ps1
Write-Host "Test case #1"
$oMarti = New-MartiChildItem -SourceFolder ".\docs" -Recurse -UrlPath ".\docs" -Filter "*" -LogPath ".\test\powershell\results\Logs"
$oMarti.description = "Sample execution"
$x = ConvertTo-Json -InputObject $oMarti
Set-Content -Path ".\test\powershell\results\marti_test01.mti" -Value $x
Write-Host "Test case #2"
$ArchiveFile = ".\test\powershell\results\marti_test02.zip"
Compress-MartiLQ -SourceFolder ".\docs" -Filter "*" -LogPath ".\test\powershell\results\Logs" -ArchiveFile $ArchiveFile
Write-Host "Test case #3"
$y = Get-MartiItem -MartiDefintiion $oMarti -Title "ckan" -Format "txt" -LogPath ".\test\powershell\results\Logs"
Write-Host "Get item Title: $($y.title)"
Write-Host "Get item Url: $($y.url)"
Write-Host "Test case #4"
$oMarti = New-MartiResource -SourcePath ".\docs\ckan.md" -LogPath ".\test\powershell\results\Logs"
$oMarti.description = "Sample execution for ckan"
$x = ConvertTo-Json -InputObject $oMarti
Set-Content -Path ".\test\powershell\results\marti_test02json.mti" -Value $x
$x = ConvertTo-Csv -InputObject $oMarti
Set-Content -Path ".\test\powershell\results\marti_test02csv.mti." -Value $x
$x = ConvertTo-Xml -As String -InputObject $oMarti -Depth 6
Set-Content -Path ".\test\powershell\results\marti_test02xml.mti" -Value $x
$x = ConvertTo-Html -InputObject $oMarti
Set-Content -Path ".\test\powershell\results\marti_test02html.mti" -Value $x
Write-Host "Test case #5"
$oMarti = New-MartiResource -SourcePath ".\docs\eror" -LogPath ".\test\powershell\results\Logs"
$oMarti.description = "Sample execution with error"
$x = ConvertTo-Json -InputObject $oMarti
Set-Content -Path ".\test\powershell\results\marti_test03.mti" -Value $x

96
test/powershell/test_MartiData1.ps1
View File

@ -1,96 +0,0 @@
. .\source\powershell\New-Marti.ps1
. .\source\powershell\ConvertFrom-Ckan.ps1
if (!(Test-Path -Path ".\test\powershell\results\data")) {
$null = New-Item -Path ".\test\powershell\results\data" -ItemType Directory
}
#$x = Get-Content -Path ".\test\results\data\bsb.csv"
$bsbFile = ".\test\powershell\results\data\bsb.csv"
$data = Import-Csv -Path $bsbFile
$columns = ($data | get-member -type NoteProperty).count
$rows = @($data).count
Write-Host "Rows: $rows Columns: $columns"
[System.Collections.ArrayList]$lattribute = @()
$oAttribute = [PSCustomObject]@{
category = "dataset"
name = "header"
function = "count"
comparison = "EQ"
value = 0
}
$lattribute += $oAttribute
$oAttribute = [PSCustomObject]@{
category = "dataset"
name = "footer"
function = "count"
comparison = "EQ"
value = 0
}
$lattribute += $oAttribute
$oAttribute = [PSCustomObject]@{
category = "dataset"
name = "rows"
function = "count"
comparison = "EQ"
value = $rows
}
$lattribute += $oAttribute
$oAttribute = [PSCustomObject]@{
category = "dataset"
name = "columns"
function = "count"
comparison = "EQ"
value = $columns
}
$lattribute += $oAttribute
$oAttribute = [PSCustomObject]@{
category = "data"
name = "BSB"
function = "sum"
comparison = "EQ"
value = 1032092
}
$lattribute += $oAttribute
$oAttribute = [PSCustomObject]@{
category = "data"
name = "BSB"
function = "unique"
comparison = "EQ"
value = $rows
}
$lattribute += $oAttribute
$uq = Get-Content $bsbFile | ConvertFrom-Csv -Header "C1", "C2" | Select-Object "C2" | Sort-Object "C2" -Unique | Group-Object -Property "C2"
$oAttribute = [PSCustomObject]@{
category = "data"
name = "Institution"
function = "unique"
comparison = "EQ"
value = $uq.Count
}
$lattribute += $oAttribute
$x = ConvertTo-Json -InputObject $lattribute
$x
#select -skip 6
#| select name

41
test/powershell/test_MartiData2.ps1
View File

@ -1,41 +0,0 @@
. .\source\powershell\New-Marti.ps1
. .\source\powershell\ConvertFrom-Ckan.ps1
. .\source\powershell\Compare-MartiResource.ps1
$bsbFile = ".\test\powershell\results\data\bsb.csv"
Write-Host ">>>>>>Test case #1"
$x = New-MartiResource -SourcePath $bsbFile -UrlPath "" -ExcludeHash -LogPath ".\test\powershell\results\Logs"
Write-Host ">>>>>>Test case #2"
$x.resources
[System.Collections.ArrayList] $Attr = Set-MartiAttribute -Attributes $x.resources[0].attributes -ACategory "dataset" -AName "records" -AFunction "count" -comparison "EQ" -value 10
$x.resources[0].attributes = Set-MartiAttribute -Attributes $Attr -ACategory "dataset" -AName "columns" -AFunction "count" -comparison "EQ" -value 8
Write-Host ">>>>>>Test case #3"
$x.resources[0].attributes
Write-Host ">>>>>>Test case #4"
$y = Compare-MartiResource -DataSource $bsbFile -Resource $x.resources[0] -LogPath ".\test\powershell\results\Logs"
$y
$covidFile = ".\test\powershell\results\data\covid-case-locations-20210920-1315.json"
Write-Host ">>>>>>Test case #5"
$x = New-MartiResource -SourcePath $covidFile -UrlPath "" -ExcludeHash -LogPath ".\test\powershell\results\Logs"
Write-Host ">>>>>>Test case #6"
$x.resources
[System.Collections.ArrayList] $Attr = Set-MartiAttribute -Attributes $x.resources[0].attributes -ACategory "dataset" -AName "records" -AFunction "count" -comparison "EQ" -value 10
$x.resources[0].attributes = Set-MartiAttribute -Attributes $Attr -ACategory "dataset" -AName "columns" -AFunction "count" -comparison "EQ" -value 8
Write-Host ">>>>>>Test case #7"
$x.resources[0].attributes
Write-Host ">>>>>>Test case #8"
$y = Compare-MartiResource -DataSource $covidFile -Resource $x.resources[0] -LogPath ".\test\powershell\results\Logs"
$y

18
test/powershell/test_MartiData3.ps1
View File

@ -1,18 +0,0 @@
. .\source\powershell\New-Marti.ps1
. .\source\powershell\ConvertFrom-Ckan.ps1
. .\source\powershell\Compare-MartiResource.ps1
$covidFile = ".\test\powershell\results\data\covid-case-locations-20210920-1315.json"
$x = New-MartiResource -SourcePath $covidFile -UrlPath "" -ExcludeHash -LogPath ".\test\powershell\results\Logs"
[System.Collections.ArrayList] $attr = Set-MartiAttribute -Attributes $x.resources[0].attributes -ACategory "dataset" -AName "records" -AFunction "count" -comparison "EQ" -value 516
$attr = Set-MartiAttribute -Attributes $attr -ACategory "dataset" -AName "columns" -AFunction "count" -comparison "EQ" -value 12
$x.resources[0].attributes = Set-MartiAttribute -Attributes $attr -ACategory "format" -AName "list" -AFunction "offset" -comparison "EQ" -value "data.monitor"
$y = Compare-MartiResource -DataSource $covidFile -Resource $x.resources[0] -LogPath ".\test\powershell\results\Logs"
$y
$attr | Get-Member

49
test/powershell/test_MartiLQ.ps1 100644
View File

@ -0,0 +1,49 @@
. .\source\powershell\MartiLQ.ps1
. .\source\powershell\Compress-MartiLQ.ps1
try {
Write-Host "Test case #1"
$oMarti = New-MartiChildItem -SourceFolder ".\docs" -Recurse -UrlPath ".\docs" -Filter "*" -LogPath ".\test\powershell\results\Logs"
$oMarti.description = "Sample execution"
$x = ConvertTo-Json -InputObject $oMarti
Set-Content -Path ".\test\powershell\results\marti_test01.mti" -Value $x
Write-Host "Test case #2"
$ArchiveFile = ".\test\powershell\results\marti_test02.zip"
Compress-MartiLQ -SourceFolder ".\docs" -Filter "*" -LogPath ".\test\powershell\results\Logs" -ArchiveFile $ArchiveFile
Write-Host "Test case #3"
$y = Get-MartiItem -MartiDefintiion $oMarti -Title "ckan" -Format "txt" -LogPath ".\test\powershell\results\Logs"
Write-Host "Get item Title: $($y.title)"
Write-Host "Get item Url: $($y.url)"
Write-Host "Test case #4"
$oMarti = New-MartiResource -SourcePath ".\docs\ckan.md" -LogPath ".\test\powershell\results\Logs"
$oMarti.description = "Sample execution for ckan"
$x = ConvertTo-Json -InputObject $oMarti
Set-Content -Path ".\test\powershell\results\marti_test02json.mti" -Value $x
$x = ConvertTo-Csv -InputObject $oMarti
Set-Content -Path ".\test\powershell\results\marti_test02csv.mti." -Value $x
$x = ConvertTo-Xml -As String -InputObject $oMarti -Depth 6
Set-Content -Path ".\test\powershell\results\marti_test02xml.mti" -Value $x
$x = ConvertTo-Html -InputObject $oMarti
Set-Content -Path ".\test\powershell\results\marti_test02html.mti" -Value $x
Write-Host "Test case #5"
$oMarti = New-MartiResource -SourcePath ".\docs\eror" -LogPath ".\test\powershell\results\Logs"
$oMarti.description = "Sample execution with error"
$x = ConvertTo-Json -InputObject $oMarti
Set-Content -Path ".\test\powershell\results\marti_test03.mti" -Value $x
}
catch {
throw
}

3
test/powershell/test_MartiCkan.ps1 → test/powershell/test_MartiLQCkan.ps1
View File

@ -1,6 +1,6 @@
. .\source\powershell\New-Marti.ps1
. .\source\powershell\MartiLQ.ps1
. .\source\powershell\ConvertFrom-Ckan.ps1
@ -23,7 +23,6 @@ Set-Content -Path ".\test\powershell\results\marti_test06.mti" -Value $x
# cases
$covid19 = "https://data.nsw.gov.au/data/api/3/action/package_show?id=3dc5dc39-40b4-4ee9-8ec6-2d862a916dcf"
#Invoke-WebRequest $covid19 -Method GET -OutFile ".\test\powershell\results\data\nsw_covid19.csv"
$covid_2 = Invoke-WebRequest $covid19
$oMarti = ConvertFrom-Ckan -InputObject $covid_2
$oMarti.description = "This data has been converted from NSW CKAN data source with URL '$covid19'"

100
test/powershell/test_MartiLQData1.ps1 100644
View File

@ -0,0 +1,100 @@
. .\source\powershell\MartiLQ.ps1
. .\source\powershell\ConvertFrom-Ckan.ps1
if (!(Test-Path -Path ".\test\powershell\results\data")) {
$null = New-Item -Path ".\test\powershell\results\data" -ItemType Directory
}
try {
$bsbFile = ".\test\powershell\results\data\BSBDirectorySep21-306.csv"
$data = Import-Csv -Path $bsbFile
$columns = ($data | get-member -type NoteProperty).count
$rows = @($data).count
Write-Host "Rows: $rows Columns: $columns"
[System.Collections.ArrayList]$lattribute = @()
$oAttribute = [PSCustomObject]@{
category = "dataset"
name = "header"
function = "count"
comparison = "EQ"
value = 0
}
$lattribute += $oAttribute
$oAttribute = [PSCustomObject]@{
category = "dataset"
name = "footer"
function = "count"
comparison = "EQ"
value = 0
}
$lattribute += $oAttribute
$oAttribute = [PSCustomObject]@{
category = "dataset"
name = "rows"
function = "count"
comparison = "EQ"
value = $rows
}
$lattribute += $oAttribute
$oAttribute = [PSCustomObject]@{
category = "dataset"
name = "columns"
function = "count"
comparison = "EQ"
value = $columns
}
$lattribute += $oAttribute
$oAttribute = [PSCustomObject]@{
category = "data"
name = "BSB"
function = "sum"
comparison = "EQ"
value = 1032092
}
$lattribute += $oAttribute
$oAttribute = [PSCustomObject]@{
category = "data"
name = "BSB"
function = "unique"
comparison = "EQ"
value = $rows
}
$lattribute += $oAttribute
$uq = Get-Content $bsbFile | ConvertFrom-Csv -Header "C1", "C2" | Select-Object "C2" | Sort-Object "C2" -Unique | Group-Object -Property "C2"
$oAttribute = [PSCustomObject]@{
category = "data"
name = "Institution"
function = "unique"
comparison = "EQ"
value = $uq.Count
}
$lattribute += $oAttribute
$x = ConvertTo-Json -InputObject $lattribute
$x
}
catch {
throw
}

48
test/powershell/test_MartiLQData2.ps1 100644
View File

@ -0,0 +1,48 @@
. .\source\powershell\MartiLQ.ps1
. .\source\powershell\MartiLQItem.ps1
. .\source\powershell\ConvertFrom-Ckan.ps1
. .\source\powershell\Compare-MartiLQResource.ps1
try {
$bsbFile = ".\test\powershell\results\data\BSBDirectorySep21-306.csv"
Write-Host ">>>>>>Test case #1"
$x = New-MartiResource -SourcePath $bsbFile -UrlPath "" -ExcludeHash -LogPath ".\test\powershell\results\Logs"
Write-Host ">>>>>>Test case #2"
$x
[System.Collections.ArrayList] $Attr = Set-MartiAttribute -Attributes $x.attributes -ACategory "dataset" -AName "records" -AFunction "count" -comparison "EQ" -value 10
$x.attributes = Set-MartiAttribute -Attributes $Attr -ACategory "dataset" -AName "columns" -AFunction "count" -comparison "EQ" -value 8
Write-Host ">>>>>>Test case #3"
$x.attributes
Write-Host ">>>>>>Test case #4"
$y = Compare-MartiResource -DataSource $bsbFile -Resource $x -LogPath ".\test\powershell\results\Logs"
$y
$covidFile = ".\test\powershell\results\data\covid-case-locations-20210920-1315.json"
Write-Host ">>>>>>Test case #5"
$x = New-MartiResource -SourcePath $covidFile -UrlPath "" -ExcludeHash -LogPath ".\test\powershell\results\Logs"
Write-Host ">>>>>>Test case #6"
$x
[System.Collections.ArrayList] $Attr = Set-MartiAttribute -Attributes $x.attributes -ACategory "dataset" -AName "records" -AFunction "count" -comparison "EQ" -value 10
$x.attributes = Set-MartiAttribute -Attributes $Attr -ACategory "dataset" -AName "columns" -AFunction "count" -comparison "EQ" -value 8
Write-Host ">>>>>>Test case #7"
$x.attributes
Write-Host ">>>>>>Test case #8"
$y = Compare-MartiResource -DataSource $covidFile -Resource $x -LogPath ".\test\powershell\results\Logs"
$y
}
catch {
throw
}

25
test/powershell/test_MartiLQData3.ps1 100644
View File

@ -0,0 +1,25 @@
. .\source\powershell\MartiLQ.ps1
. .\source\powershell\MartiLQItem.ps1
. .\source\powershell\ConvertFrom-Ckan.ps1
. .\source\powershell\Compare-MartiLQResource.ps1
try {
$covidFile = ".\test\powershell\results\data\covid-case-locations-20210920-1315.json"
$x = New-MartiResource -SourcePath $covidFile -UrlPath "" -ExcludeHash -LogPath ".\test\powershell\results\Logs"
[System.Collections.ArrayList] $attr = Set-MartiAttribute -Attributes $x.attributes -ACategory "dataset" -AName "records" -AFunction "count" -comparison "EQ" -value 516
$attr = Set-MartiAttribute -Attributes $attr -ACategory "dataset" -AName "columns" -AFunction "count" -comparison "EQ" -value 12
$x.attributes = Set-MartiAttribute -Attributes $attr -ACategory "format" -AName "list" -AFunction "offset" -comparison "EQ" -value "data.monitor"
$y = Compare-MartiResource -DataSource $covidFile -Resource $x -LogPath ".\test\powershell\results\Logs"
$y
$attr | Get-Member
}
catch {
throw
}

30
test/private.pem 100644
View File

@ -0,0 +1,30 @@
-----BEGIN RSA PRIVATE KEY-----
Proc-Type: 4,ENCRYPTED
DEK-Info: DES-EDE3-CBC,43D90161F4DF49CC
RLyZTiOgXqyrj8/ecWIKf2oZviplxQCqfhr9s7VfuvPlQwDVroi91d9AnrJvOCFL
yyhHTIQxDj0NoEIZ+DDToo2qvoHlmStIpYs1WXvFRZnVsA2KUPmcg9vaRlTb4n4i
XU9c/uWDDvdI2k+BuFoeyWaPnn9gBS9y7sPcvBwpDHMMVQ9Quv5q74c9O99Rtb/p
gpKWAEO6fLRw8xGyaMnoGCMQbcaa/af6hSuvnFkrJwlkp9fYyfz9bWwwznjwsRBT
QC+T5pTp5ZgQ7J/d0Kr21RrBFUlUDtw7C3y5UCY54ajAPuHpoBEXutOGNbzuHNFq
sA1i9iu7L2TBT2gp4XlCtnq9RDqAirn/heNDlzJWYAi00a4yratmQepq4n1M3L1e
MMb61PNQckvyeWjwLc1QoxK+spyxd9ZjriEXTv7/UjnNgfWXep3KqR4QNRO6u6xq
7Bg/87kY1hpLHNLgVhp6laKe68pvxeO3X0OMKw1HOGI9CxdYi92GYXsNgQULscnp
ukusKVPHQOUCQe0YzkVcIjrmJ3/EMRvJ3cXRvxmZfal2mJNOsuBwEQunkJRwDBKR
WpBArhNRToGG6gCe4+i9vXzyXjYg+szs8XyONY0AzTW+8HXEuBddp+otke7fSsYp
QIYCoinNjnXyaTuNHIlFl/9/FXacjaBAS+5UVEbYZ80oYPzcqMhbaUl2be9UDxQn
ThjnYoUo+zYGrCqvqVLf/uTM0Wgd/Ac3jZlS86vsHmYi2LXjsWRdSuD11w25pB9r
hSrt2n6hRavB8XxOoni6K0lBKTOI4KyxVFDFD2e5aUj2pYpOkG+sjjGM/YQw16Uo
yA1IiiOUaBGWZ/EnFhHLD9oeOyQ5LD3F6A2IHSfEYK0JSZhlZKzL1GGrlrgGtY3O
gYKijUhjZpGRqlTKO7jG3v6HzQia4sw1mi0C8IlNXZIGqa6HcaCidi3CsFdcKoiB
bye0xIIQJPAMR0ybi6ptRAKjINGGlIc1pAfxWq4lu4MDp9zvpo52GaHxvSapaye8
JnXDLCfa5gxBBr4WOz7S49gb3UP+6JZUGGKTWsd/RdZNDqmR9GMlTjVx6anMLEwQ
YvWyqbPfuzHXe6+XCDYinBTv35eTbZC28FpwvBXniZ8grM0jxqSAi24KQk6f0Ynx
KtWwgGsQLy+5BpX1PZAGRnKZBGWOsjOAOgvn1ZWTEzhu8woci6On8ZtgXuLmemws
uQdjDCl7B7/fHIElPMg/3FZkYg4cTooazNvBnszbvubkkNFbVFFLn/e3RMSsAuJy
FURqYoXp3DkcF1931oJeBjno3MblgQ/6nmWrt/Z6PHd6FVbJqeEEExDWdtz5nWvf
BmFK/TFhrn8zeUGbxtZjUhHGTHC9a36ybeFsc04i8yE/8vbpYguEHGBPndro6xpK
xIEkZV/EfAgwbu3HiRlVCgLFWyKGz+flG6A1pMeKsbeFmdz4WieuFOjxQtL5Qhek
obyPH3XFWv5xFD3TxxMgRKFUXsoWCjAHE5RXi7F0+YJd9GeKqFutohHJyCOFc6L8
PZ8b6e7wkh8jqOKwJnlZXDYooI5jvXNWr+KJb96zYWYEHGIkdmvmdg==
-----END RSA PRIVATE KEY-----

9
test/public.pem 100644
View File

@ -0,0 +1,9 @@
-----BEGIN PUBLIC KEY-----
MIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEAmpZRUYQql7FCIPB67IbS
jB15jDRkVARdIrzdnfEx3rXG3l5YI2+oilN8sLaS+oot/PYNQICOHOe1w6hhWnzI
eviX2PgA1bf9lNDHYSN3R70F2b1CbrFIA/WXNv58s/yjg5aIjD0lfQ/o1KjAIsUc
cA8VKkW8sB0SidjDdHSsyWWF0LKiRGFIAfLgE/spG06zT4M1fB9AgWmqrcCOwW6s
DC5dFXnW5ZnaAD5xD8gZdAM/i7JybCOXOJKZLiQNXGLMYp1iFk8fSeuiyJRVIgqI
mIYbbPkKaYNBIi9LmUg67HcSODL8uZDtogiRpIZV72NjhsK4iGWvoY9EcKaMNMgS
cwIDAQAB
-----END PUBLIC KEY-----

39
test/python/test_MartiLQ1.py 100644
View File

@ -0,0 +1,39 @@
import os
import json
import sys
import csv
import zipfile
sys.path.insert(0, "./source/python/client")
from martiLQ import *
os.environ["MARTILQ_LOGPATH"] = "./test/python/results/logs"
print("Python test case #1")
mlq = martiLQ()
mlq.LoadConfig()
oMarti = mlq.NewMartiDefinition()
mlq.NewMartiChildItem(SourceFolder= "./docs/*", UrlPath="./docs" , ExcludeHash=False, ExtendAttributes=True)
oMarti["description"] = "Sample execution #1"
print("Save martiLQ definition #1")
mlq.Save("./test/python/results/DocsPlain1.mti")
print("Save martiLQ definition #2")
oMarti["description"] = "Sample execution #2"
jsonFile = open("./test/python/results/DocsPlain2.mti", "w")
jsonFile.write(json.dumps(oMarti, indent=5))
jsonFile.close()
print("Base sample mti written: DocsPlain2.mti")
print("Load martiLQ definition #1")
mlq.Load("./test/python/results/DocsPlain1.mti")
oMarti = mlq.Get()
print("Definition description is: {}".format(oMarti["description"]))
mlq.CloseLog()
print("Completed Python test case #1")