parent
463328245b
commit
ed84028bd4
|
|
@ -0,0 +1,418 @@
|
||||||
|
{
|
||||||
|
"content-type": "application/vnd.martilq.json",
|
||||||
|
"title": "SampleFetchBSB",
|
||||||
|
"uid": "08e22a4d-5a11-4b19-9172-feb36601009e",
|
||||||
|
"description": "",
|
||||||
|
"modified": "2021-10-23",
|
||||||
|
"publisher": "meerkat",
|
||||||
|
"contactPoint": "",
|
||||||
|
"accessLevel": "Public",
|
||||||
|
"rights": "Publish",
|
||||||
|
"tags": ["BSB", "Australia"],
|
||||||
|
"license": "",
|
||||||
|
"state": "active",
|
||||||
|
"batch": 1.0,
|
||||||
|
"describedBy": "",
|
||||||
|
"landingPage": "",
|
||||||
|
"theme": "",
|
||||||
|
"resources": [
|
||||||
|
{
|
||||||
|
"title": "BSBDirectoryAug21-305",
|
||||||
|
"uid": "59f1c10e-a71b-45a4-9205-2b651cf04304",
|
||||||
|
"documentName": "BSBDirectoryAug21-305.csv",
|
||||||
|
"issuedDate": "2021-10-23T18:37:58",
|
||||||
|
"modified": "2021-10-23T18:37:46",
|
||||||
|
"state": "active",
|
||||||
|
"author": "Meerkat@merebox.com",
|
||||||
|
"length": 1352016,
|
||||||
|
"hash": {
|
||||||
|
"algo": "SHA256",
|
||||||
|
"value": "26e40760989ff8a314cda70628c38b66d0d565ea8ad677b81b83780e33984104",
|
||||||
|
"signed": false
|
||||||
|
},
|
||||||
|
"description": "",
|
||||||
|
"url": "ftp://bsb.hostedftp.com/~auspaynetftp/BSB/BSBDirectoryAug21-305.csv",
|
||||||
|
"version": "",
|
||||||
|
"content-type": "text/csv",
|
||||||
|
"compression": null,
|
||||||
|
"encryption": null,
|
||||||
|
"attributes": [
|
||||||
|
{
|
||||||
|
"category": "dataset",
|
||||||
|
"name": "header",
|
||||||
|
"function": "count",
|
||||||
|
"comparison": "NA",
|
||||||
|
"value": 1
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"category": "dataset",
|
||||||
|
"name": "footer",
|
||||||
|
"function": "count",
|
||||||
|
"comparison": "NA",
|
||||||
|
"value": 0
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"category": "format",
|
||||||
|
"name": "separator",
|
||||||
|
"function": "value",
|
||||||
|
"comparison": "NA",
|
||||||
|
"value": ","
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"category": "format",
|
||||||
|
"name": "columns",
|
||||||
|
"function": "value",
|
||||||
|
"comparison": "NA",
|
||||||
|
"value": ","
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"category": "dataset",
|
||||||
|
"name": "records",
|
||||||
|
"function": "count",
|
||||||
|
"comparison": "EQ",
|
||||||
|
"value": 15055
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"category": "dataset",
|
||||||
|
"name": "columns",
|
||||||
|
"function": "count",
|
||||||
|
"comparison": "EQ",
|
||||||
|
"value": 8
|
||||||
|
}
|
||||||
|
]
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"title": "BSBDirectoryAug21-305",
|
||||||
|
"uid": "cd79772f-ff33-4fb6-b90e-94118f80fd69",
|
||||||
|
"documentName": "BSBDirectoryAug21-305.txt",
|
||||||
|
"issuedDate": "2021-10-23T18:37:58",
|
||||||
|
"modified": "2021-10-23T18:37:48",
|
||||||
|
"state": "active",
|
||||||
|
"author": "Meerkat@merebox.com",
|
||||||
|
"length": 1701441,
|
||||||
|
"hash": {
|
||||||
|
"algo": "SHA256",
|
||||||
|
"value": "799f937f33b0284f0493063f51117f154c415a9554b8ba7d07f47416d75208a3",
|
||||||
|
"signed": false
|
||||||
|
},
|
||||||
|
"description": "",
|
||||||
|
"url": "ftp://bsb.hostedftp.com/~auspaynetftp/BSB/BSBDirectoryAug21-305.txt",
|
||||||
|
"version": "",
|
||||||
|
"content-type": "text/plain",
|
||||||
|
"compression": null,
|
||||||
|
"encryption": null,
|
||||||
|
"attributes": [
|
||||||
|
{
|
||||||
|
"category": "dataset",
|
||||||
|
"name": "header",
|
||||||
|
"function": "count",
|
||||||
|
"comparison": "NA",
|
||||||
|
"value": 1
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"category": "dataset",
|
||||||
|
"name": "footer",
|
||||||
|
"function": "count",
|
||||||
|
"comparison": "NA",
|
||||||
|
"value": 0
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"category": "format",
|
||||||
|
"name": "separator",
|
||||||
|
"function": "value",
|
||||||
|
"comparison": "NA",
|
||||||
|
"value": ","
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"category": "format",
|
||||||
|
"name": "columns",
|
||||||
|
"function": "value",
|
||||||
|
"comparison": "NA",
|
||||||
|
"value": ","
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"category": "dataset",
|
||||||
|
"name": "records",
|
||||||
|
"function": "count",
|
||||||
|
"comparison": "EQ",
|
||||||
|
"value": 15057
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"category": "dataset",
|
||||||
|
"name": "columns",
|
||||||
|
"function": "count",
|
||||||
|
"comparison": "EQ",
|
||||||
|
"value": 1
|
||||||
|
}
|
||||||
|
]
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"title": "BSBDirectoryJul21-304",
|
||||||
|
"uid": "4f57b82a-6b6a-4612-8d4e-67289b7795e1",
|
||||||
|
"documentName": "BSBDirectoryJul21-304.csv",
|
||||||
|
"issuedDate": "2021-10-23T18:37:58",
|
||||||
|
"modified": "2021-10-23T18:37:51",
|
||||||
|
"state": "active",
|
||||||
|
"author": "Meerkat@merebox.com",
|
||||||
|
"length": 1351065,
|
||||||
|
"hash": {
|
||||||
|
"algo": "SHA256",
|
||||||
|
"value": "5306b8baaafaf60600b645f3514dde3f654ac2b0bdbae96b0d0501b18488b1b5",
|
||||||
|
"signed": false
|
||||||
|
},
|
||||||
|
"description": "",
|
||||||
|
"url": "ftp://bsb.hostedftp.com/~auspaynetftp/BSB/BSBDirectoryJul21-304.csv",
|
||||||
|
"version": "",
|
||||||
|
"content-type": "text/csv",
|
||||||
|
"compression": null,
|
||||||
|
"encryption": null,
|
||||||
|
"attributes": [
|
||||||
|
{
|
||||||
|
"category": "dataset",
|
||||||
|
"name": "header",
|
||||||
|
"function": "count",
|
||||||
|
"comparison": "NA",
|
||||||
|
"value": 1
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"category": "dataset",
|
||||||
|
"name": "footer",
|
||||||
|
"function": "count",
|
||||||
|
"comparison": "NA",
|
||||||
|
"value": 0
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"category": "format",
|
||||||
|
"name": "separator",
|
||||||
|
"function": "value",
|
||||||
|
"comparison": "NA",
|
||||||
|
"value": ","
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"category": "format",
|
||||||
|
"name": "columns",
|
||||||
|
"function": "value",
|
||||||
|
"comparison": "NA",
|
||||||
|
"value": ","
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"category": "dataset",
|
||||||
|
"name": "records",
|
||||||
|
"function": "count",
|
||||||
|
"comparison": "EQ",
|
||||||
|
"value": 15044
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"category": "dataset",
|
||||||
|
"name": "columns",
|
||||||
|
"function": "count",
|
||||||
|
"comparison": "EQ",
|
||||||
|
"value": 8
|
||||||
|
}
|
||||||
|
]
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"title": "BSBDirectoryJul21-304",
|
||||||
|
"uid": "e8bbdfc4-ab46-465c-af97-39e4e4a3e36c",
|
||||||
|
"documentName": "BSBDirectoryJul21-304.txt",
|
||||||
|
"issuedDate": "2021-10-23T18:37:58",
|
||||||
|
"modified": "2021-10-23T18:37:53",
|
||||||
|
"state": "active",
|
||||||
|
"author": "Meerkat@merebox.com",
|
||||||
|
"length": 1700198,
|
||||||
|
"hash": {
|
||||||
|
"algo": "SHA256",
|
||||||
|
"value": "60d14b4ae28c8e91499c73e385504ea904134c0fa348af25a7baf58eb9eb8390",
|
||||||
|
"signed": false
|
||||||
|
},
|
||||||
|
"description": "",
|
||||||
|
"url": "ftp://bsb.hostedftp.com/~auspaynetftp/BSB/BSBDirectoryJul21-304.txt",
|
||||||
|
"version": "",
|
||||||
|
"content-type": "text/plain",
|
||||||
|
"compression": null,
|
||||||
|
"encryption": null,
|
||||||
|
"attributes": [
|
||||||
|
{
|
||||||
|
"category": "dataset",
|
||||||
|
"name": "header",
|
||||||
|
"function": "count",
|
||||||
|
"comparison": "NA",
|
||||||
|
"value": 1
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"category": "dataset",
|
||||||
|
"name": "footer",
|
||||||
|
"function": "count",
|
||||||
|
"comparison": "NA",
|
||||||
|
"value": 0
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"category": "format",
|
||||||
|
"name": "separator",
|
||||||
|
"function": "value",
|
||||||
|
"comparison": "NA",
|
||||||
|
"value": ","
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"category": "format",
|
||||||
|
"name": "columns",
|
||||||
|
"function": "value",
|
||||||
|
"comparison": "NA",
|
||||||
|
"value": ","
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"category": "dataset",
|
||||||
|
"name": "records",
|
||||||
|
"function": "count",
|
||||||
|
"comparison": "EQ",
|
||||||
|
"value": 15046
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"category": "dataset",
|
||||||
|
"name": "columns",
|
||||||
|
"function": "count",
|
||||||
|
"comparison": "EQ",
|
||||||
|
"value": 1
|
||||||
|
}
|
||||||
|
]
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"title": "BSBDirectorySep21-306",
|
||||||
|
"uid": "08d11601-54a6-4fb0-b694-e2d2c2a7eb32",
|
||||||
|
"documentName": "BSBDirectorySep21-306.csv",
|
||||||
|
"issuedDate": "2021-10-23T18:37:58",
|
||||||
|
"modified": "2021-10-23T18:37:55",
|
||||||
|
"state": "active",
|
||||||
|
"author": "Meerkat@merebox.com",
|
||||||
|
"length": 1351281,
|
||||||
|
"hash": {
|
||||||
|
"algo": "SHA256",
|
||||||
|
"value": "4f5c97799f6006247fad64f9a5acc425e79f31342cbbbdaaba2995af16ac56b4",
|
||||||
|
"signed": false
|
||||||
|
},
|
||||||
|
"description": "",
|
||||||
|
"url": "ftp://bsb.hostedftp.com/~auspaynetftp/BSB/BSBDirectorySep21-306.csv",
|
||||||
|
"version": "",
|
||||||
|
"content-type": "text/csv",
|
||||||
|
"compression": null,
|
||||||
|
"encryption": null,
|
||||||
|
"attributes": [
|
||||||
|
{
|
||||||
|
"category": "dataset",
|
||||||
|
"name": "header",
|
||||||
|
"function": "count",
|
||||||
|
"comparison": "NA",
|
||||||
|
"value": 1
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"category": "dataset",
|
||||||
|
"name": "footer",
|
||||||
|
"function": "count",
|
||||||
|
"comparison": "NA",
|
||||||
|
"value": 0
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"category": "format",
|
||||||
|
"name": "separator",
|
||||||
|
"function": "value",
|
||||||
|
"comparison": "NA",
|
||||||
|
"value": ","
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"category": "format",
|
||||||
|
"name": "columns",
|
||||||
|
"function": "value",
|
||||||
|
"comparison": "NA",
|
||||||
|
"value": ","
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"category": "dataset",
|
||||||
|
"name": "records",
|
||||||
|
"function": "count",
|
||||||
|
"comparison": "EQ",
|
||||||
|
"value": 15050
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"category": "dataset",
|
||||||
|
"name": "columns",
|
||||||
|
"function": "count",
|
||||||
|
"comparison": "EQ",
|
||||||
|
"value": 8
|
||||||
|
}
|
||||||
|
]
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"title": "BSBDirectorySep21-306",
|
||||||
|
"uid": "a4a01312-a254-4ba9-b9c7-294b1589a2e1",
|
||||||
|
"documentName": "BSBDirectorySep21-306.txt",
|
||||||
|
"issuedDate": "2021-10-23T18:37:58",
|
||||||
|
"modified": "2021-10-23T18:37:58",
|
||||||
|
"state": "active",
|
||||||
|
"author": "Meerkat@merebox.com",
|
||||||
|
"length": 1700876,
|
||||||
|
"hash": {
|
||||||
|
"algo": "SHA256",
|
||||||
|
"value": "dd830996c087b5b781a8d4d2bf7ec8da2b0de642a338ffbadea0d8d3834e2232",
|
||||||
|
"signed": false
|
||||||
|
},
|
||||||
|
"description": "",
|
||||||
|
"url": "ftp://bsb.hostedftp.com/~auspaynetftp/BSB/BSBDirectorySep21-306.txt",
|
||||||
|
"version": "",
|
||||||
|
"content-type": "text/plain",
|
||||||
|
"compression": null,
|
||||||
|
"encryption": null,
|
||||||
|
"attributes": [
|
||||||
|
{
|
||||||
|
"category": "dataset",
|
||||||
|
"name": "header",
|
||||||
|
"function": "count",
|
||||||
|
"comparison": "NA",
|
||||||
|
"value": 1
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"category": "dataset",
|
||||||
|
"name": "footer",
|
||||||
|
"function": "count",
|
||||||
|
"comparison": "NA",
|
||||||
|
"value": 0
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"category": "format",
|
||||||
|
"name": "separator",
|
||||||
|
"function": "value",
|
||||||
|
"comparison": "NA",
|
||||||
|
"value": ","
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"category": "format",
|
||||||
|
"name": "columns",
|
||||||
|
"function": "value",
|
||||||
|
"comparison": "NA",
|
||||||
|
"value": ","
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"category": "dataset",
|
||||||
|
"name": "records",
|
||||||
|
"function": "count",
|
||||||
|
"comparison": "EQ",
|
||||||
|
"value": 15052
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"category": "dataset",
|
||||||
|
"name": "columns",
|
||||||
|
"function": "count",
|
||||||
|
"comparison": "EQ",
|
||||||
|
"value": 1
|
||||||
|
}
|
||||||
|
]
|
||||||
|
}
|
||||||
|
],
|
||||||
|
"custom": [
|
||||||
|
{
|
||||||
|
"extension": "software",
|
||||||
|
"softwareName": "MARTILQREFERENCE",
|
||||||
|
"author": "Meerkat@merebox.com",
|
||||||
|
"version": "0.0.1"
|
||||||
|
}
|
||||||
|
]
|
||||||
|
}
|
||||||
|
|
@ -0,0 +1,223 @@
|
||||||
|
{
|
||||||
|
"content-type": "application/vnd.martilq.json",
|
||||||
|
"title": "SampleFetchBSB",
|
||||||
|
"uid": "08e22a4d-5a11-4b19-9172-feb36601009e",
|
||||||
|
"description": "",
|
||||||
|
"modified": "2021-10-23",
|
||||||
|
"publisher": "meerkat",
|
||||||
|
"contactPoint": "",
|
||||||
|
"accessLevel": "Public",
|
||||||
|
"rights": "Publish",
|
||||||
|
"tags": ["BSB", "Australia"],
|
||||||
|
"license": "",
|
||||||
|
"state": "active",
|
||||||
|
"batch": 1.0,
|
||||||
|
"describedBy": "",
|
||||||
|
"landingPage": "",
|
||||||
|
"theme": "",
|
||||||
|
"resources": [
|
||||||
|
{
|
||||||
|
"title": "BSBDirectoryAug21-305",
|
||||||
|
"uid": "59f1c10e-a71b-45a4-9205-2b651cf04304",
|
||||||
|
"documentName": "BSBDirectoryAug21-305.csv",
|
||||||
|
"issuedDate": "2021-10-23T18:37:58",
|
||||||
|
"modified": "2021-10-23T18:37:46",
|
||||||
|
"state": "active",
|
||||||
|
"author": "Meerkat@merebox.com",
|
||||||
|
"length": 1352016,
|
||||||
|
"hash": {
|
||||||
|
"algo": "SHA256",
|
||||||
|
"value": "26e40760989ff8a314cda70628c38b66d0d565ea8ad677b81b83780e33984104",
|
||||||
|
"signed": false
|
||||||
|
},
|
||||||
|
"description": "",
|
||||||
|
"url": "http://apnedata.merebox.com.s3.ap-southeast-2.amazonaws.com/au/bsb/BSBDirectoryAug21-305.csv",
|
||||||
|
"version": "",
|
||||||
|
"content-type": "text/csv",
|
||||||
|
"compression": null,
|
||||||
|
"encryption": null,
|
||||||
|
"attributes": [
|
||||||
|
{
|
||||||
|
"category": "dataset",
|
||||||
|
"name": "header",
|
||||||
|
"function": "count",
|
||||||
|
"comparison": "NA",
|
||||||
|
"value": 1
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"category": "dataset",
|
||||||
|
"name": "footer",
|
||||||
|
"function": "count",
|
||||||
|
"comparison": "NA",
|
||||||
|
"value": 0
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"category": "format",
|
||||||
|
"name": "separator",
|
||||||
|
"function": "value",
|
||||||
|
"comparison": "NA",
|
||||||
|
"value": ","
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"category": "format",
|
||||||
|
"name": "columns",
|
||||||
|
"function": "value",
|
||||||
|
"comparison": "NA",
|
||||||
|
"value": ","
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"category": "dataset",
|
||||||
|
"name": "records",
|
||||||
|
"function": "count",
|
||||||
|
"comparison": "EQ",
|
||||||
|
"value": 15055
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"category": "dataset",
|
||||||
|
"name": "columns",
|
||||||
|
"function": "count",
|
||||||
|
"comparison": "EQ",
|
||||||
|
"value": 8
|
||||||
|
}
|
||||||
|
]
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"title": "BSBDirectoryJul21-304",
|
||||||
|
"uid": "4f57b82a-6b6a-4612-8d4e-67289b7795e1",
|
||||||
|
"documentName": "BSBDirectoryJul21-304.csv",
|
||||||
|
"issuedDate": "2021-10-23T18:37:58",
|
||||||
|
"modified": "2021-10-23T18:37:51",
|
||||||
|
"state": "active",
|
||||||
|
"author": "Meerkat@merebox.com",
|
||||||
|
"length": 1351065,
|
||||||
|
"hash": {
|
||||||
|
"algo": "SHA256",
|
||||||
|
"value": "5306b8baaafaf60600b645f3514dde3f654ac2b0bdbae96b0d0501b18488b1b5",
|
||||||
|
"signed": false
|
||||||
|
},
|
||||||
|
"description": "",
|
||||||
|
"url": "http://apnedata.merebox.com.s3.ap-southeast-2.amazonaws.com/au/bsb/BSBDirectoryJul21-304.csv",
|
||||||
|
"version": "",
|
||||||
|
"content-type": "text/csv",
|
||||||
|
"compression": null,
|
||||||
|
"encryption": null,
|
||||||
|
"attributes": [
|
||||||
|
{
|
||||||
|
"category": "dataset",
|
||||||
|
"name": "header",
|
||||||
|
"function": "count",
|
||||||
|
"comparison": "NA",
|
||||||
|
"value": 1
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"category": "dataset",
|
||||||
|
"name": "footer",
|
||||||
|
"function": "count",
|
||||||
|
"comparison": "NA",
|
||||||
|
"value": 0
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"category": "format",
|
||||||
|
"name": "separator",
|
||||||
|
"function": "value",
|
||||||
|
"comparison": "NA",
|
||||||
|
"value": ","
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"category": "format",
|
||||||
|
"name": "columns",
|
||||||
|
"function": "value",
|
||||||
|
"comparison": "NA",
|
||||||
|
"value": ","
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"category": "dataset",
|
||||||
|
"name": "records",
|
||||||
|
"function": "count",
|
||||||
|
"comparison": "EQ",
|
||||||
|
"value": 15044
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"category": "dataset",
|
||||||
|
"name": "columns",
|
||||||
|
"function": "count",
|
||||||
|
"comparison": "EQ",
|
||||||
|
"value": 8
|
||||||
|
}
|
||||||
|
]
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"title": "BSBDirectorySep21-306",
|
||||||
|
"uid": "08d11601-54a6-4fb0-b694-e2d2c2a7eb32",
|
||||||
|
"documentName": "BSBDirectorySep21-306.csv",
|
||||||
|
"issuedDate": "2021-10-23T18:37:58",
|
||||||
|
"modified": "2021-10-23T18:37:55",
|
||||||
|
"state": "active",
|
||||||
|
"author": "Meerkat@merebox.com",
|
||||||
|
"length": 1351281,
|
||||||
|
"hash": {
|
||||||
|
"algo": "SHA256",
|
||||||
|
"value": "4f5c97799f6006247fad64f9a5acc425e79f31342cbbbdaaba2995af16ac56b4",
|
||||||
|
"signed": false
|
||||||
|
},
|
||||||
|
"description": "",
|
||||||
|
"url": "http://apnedata.merebox.com.s3.ap-southeast-2.amazonaws.com/au/bsb/BSBDirectorySep21-306.csv",
|
||||||
|
"version": "",
|
||||||
|
"content-type": "text/csv",
|
||||||
|
"compression": null,
|
||||||
|
"encryption": null,
|
||||||
|
"attributes": [
|
||||||
|
{
|
||||||
|
"category": "dataset",
|
||||||
|
"name": "header",
|
||||||
|
"function": "count",
|
||||||
|
"comparison": "NA",
|
||||||
|
"value": 1
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"category": "dataset",
|
||||||
|
"name": "footer",
|
||||||
|
"function": "count",
|
||||||
|
"comparison": "NA",
|
||||||
|
"value": 0
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"category": "format",
|
||||||
|
"name": "separator",
|
||||||
|
"function": "value",
|
||||||
|
"comparison": "NA",
|
||||||
|
"value": ","
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"category": "format",
|
||||||
|
"name": "columns",
|
||||||
|
"function": "value",
|
||||||
|
"comparison": "NA",
|
||||||
|
"value": ","
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"category": "dataset",
|
||||||
|
"name": "records",
|
||||||
|
"function": "count",
|
||||||
|
"comparison": "EQ",
|
||||||
|
"value": 15050
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"category": "dataset",
|
||||||
|
"name": "columns",
|
||||||
|
"function": "count",
|
||||||
|
"comparison": "EQ",
|
||||||
|
"value": 8
|
||||||
|
}
|
||||||
|
]
|
||||||
|
}
|
||||||
|
],
|
||||||
|
"custom": [
|
||||||
|
{
|
||||||
|
"extension": "software",
|
||||||
|
"softwareName": "MARTILQREFERENCE",
|
||||||
|
"author": "Meerkat@merebox.com",
|
||||||
|
"version": "0.0.1"
|
||||||
|
}
|
||||||
|
]
|
||||||
|
}
|
||||||
|
|
@ -152,7 +152,7 @@ foreach($file in Get-ChildItem $localDirectory)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
$oResource = New-MartiResource -SourcePath $zipFile -UrlPath $localDirectory -LogPath ".\test\Logs" -ExtendAttributes
|
$oResource = New-MartiResource -SourcePath $zipFile -UrlPath $localDirectory -LogPath ".\test\Logs" -ExtendAttributes
|
||||||
Set-AttributeValueString -Attributes $oResource.attributes -Key "compression" -Category "format" -Function "algorithm" -Value "WINZIP"
|
Set-AttributeValueString -Attributes $oResource.attributes -Key "compression" -Category "format" -Function "algo" -Value "WINZIP"
|
||||||
$oMarti.resources += $oResource
|
$oMarti.resources += $oResource
|
||||||
|
|
||||||
$fileJson = Join-Path -Path $localDirectory -ChildPath "MartiBSBZip.mti"
|
$fileJson = Join-Path -Path $localDirectory -ChildPath "MartiBSBZip.mti"
|
||||||
|
|
@ -224,7 +224,7 @@ Compress-7Zip -Path $noticeFile -ArchiveFileName $zipFile -Append -Password $sec
|
||||||
$oResource = New-MartiResource -SourcePath $zipFile -UrlPath $localDirectory -LogPath ".\test\Logs" -ExtendAttributes
|
$oResource = New-MartiResource -SourcePath $zipFile -UrlPath $localDirectory -LogPath ".\test\Logs" -ExtendAttributes
|
||||||
$oResource.compression = "7ZIP"
|
$oResource.compression = "7ZIP"
|
||||||
$oResource.encryption = New-Encryption -Algorithm "Passphrase" -Value $secret
|
$oResource.encryption = New-Encryption -Algorithm "Passphrase" -Value $secret
|
||||||
Set-AttributeValueString -Attributes $oResource.attributes -Key "compression" -Category "format" -Function "algorithm" -Value "7ZIP"
|
Set-AttributeValueString -Attributes $oResource.attributes -Key "compression" -Category "format" -Function "algo" -Value "7ZIP"
|
||||||
$oMarti.resources += $oResource
|
$oMarti.resources += $oResource
|
||||||
|
|
||||||
$fileJson = Join-Path -Path $localDirectory -ChildPath "MartiBSBSecure.mti"
|
$fileJson = Join-Path -Path $localDirectory -ChildPath "MartiBSBSecure.mti"
|
||||||
|
|
|
||||||
|
|
@ -161,7 +161,7 @@ $oResource.encryption = New-Encryption -Algorithm "PKI" -Value $($encryptedSecre
|
||||||
|
|
||||||
Write-Debug "Secret: $secret"
|
Write-Debug "Secret: $secret"
|
||||||
|
|
||||||
Set-AttributeValueString -Attributes $oResource.attributes -Key "compression" -Category "format" -Function "algorithm" -Value "7ZIP"
|
Set-AttributeValueString -Attributes $oResource.attributes -Key "compression" -Category "format" -Function "algo" -Value "7ZIP"
|
||||||
$oMarti.resources += $oResource
|
$oMarti.resources += $oResource
|
||||||
|
|
||||||
$fileJson = Join-Path -Path $localDirectory -ChildPath "MartiBSB7ZipPKI.mti"
|
$fileJson = Join-Path -Path $localDirectory -ChildPath "MartiBSB7ZipPKI.mti"
|
||||||
|
|
|
||||||
|
|
@ -0,0 +1,56 @@
|
||||||
|
|
||||||
|
import os
|
||||||
|
import sys
|
||||||
|
import csv
|
||||||
|
|
||||||
|
sys.path.insert(0, "../../../source/python/client")
|
||||||
|
from martiLQ import *
|
||||||
|
|
||||||
|
test_directory = "./test/fetch_ftp"
|
||||||
|
os.environ["MARTILQ_LOGPATH"] = os.path.join(test_directory, "logs")
|
||||||
|
|
||||||
|
if not os.path.exists("./test"):
|
||||||
|
os.mkdir("./test")
|
||||||
|
|
||||||
|
if not os.path.exists(test_directory):
|
||||||
|
os.mkdir(test_directory)
|
||||||
|
|
||||||
|
print("Creating martiLQ definition")
|
||||||
|
mlq = martiLQ()
|
||||||
|
mlq.LoadConfig()
|
||||||
|
print("Loading definition json")
|
||||||
|
mlq.Load("../BSBDirectoryFtp.json")
|
||||||
|
print("Fetching files")
|
||||||
|
fetched_files, fetch_error = mlq.Fetch(test_directory)
|
||||||
|
|
||||||
|
if len(fetched_files) < 0:
|
||||||
|
raise Exception("No resource files fetched")
|
||||||
|
else:
|
||||||
|
print("Fetched {} files".format(len(fetched_files)))
|
||||||
|
|
||||||
|
if len(fetch_error) > 0:
|
||||||
|
raise Exception("Some resources not fetched")
|
||||||
|
|
||||||
|
print("Generate the self value, overriding existing")
|
||||||
|
oMarti = mlq.NewMartiDefinition()
|
||||||
|
for full_fileName in fetched_files:
|
||||||
|
if os.path.isfile(full_fileName):
|
||||||
|
oResource = mlq.NewMartiLQResource(full_fileName, "", False, True)
|
||||||
|
oMarti["resources"].append(oResource)
|
||||||
|
print("Perform validation test")
|
||||||
|
lqresults, testError = mlq.TestMartiDefinition("../BSBDirectoryFtp.json")
|
||||||
|
|
||||||
|
testfile = open("./test/LoadQualityTest_Ftp.csv", "w+", newline ="")
|
||||||
|
with testfile:
|
||||||
|
lqwriter = csv.writer(testfile)
|
||||||
|
lqwriter.writerows(lqresults)
|
||||||
|
|
||||||
|
if testError > 0:
|
||||||
|
print("MISMATCH DETECTED")
|
||||||
|
else:
|
||||||
|
print("RECONCILED")
|
||||||
|
|
||||||
|
mlq.CloseLog()
|
||||||
|
|
||||||
|
print("Test completed: SampleFetchFtpBsb.py")
|
||||||
|
|
||||||
|
|
@ -0,0 +1,56 @@
|
||||||
|
|
||||||
|
import os
|
||||||
|
import sys
|
||||||
|
import csv
|
||||||
|
|
||||||
|
sys.path.insert(0, "../../../source/python/client")
|
||||||
|
from martiLQ import *
|
||||||
|
|
||||||
|
test_directory = "./test/fetch_http"
|
||||||
|
os.environ["MARTILQ_LOGPATH"] = os.path.join(test_directory, "logs")
|
||||||
|
|
||||||
|
if not os.path.exists("./test"):
|
||||||
|
os.mkdir("./test")
|
||||||
|
|
||||||
|
if not os.path.exists(test_directory):
|
||||||
|
os.mkdir(test_directory)
|
||||||
|
|
||||||
|
print("Creating martiLQ definition")
|
||||||
|
mlq = martiLQ()
|
||||||
|
mlq.LoadConfig()
|
||||||
|
print("Loading definition json")
|
||||||
|
mlq.Load("../BSBDirectoryHttp.json")
|
||||||
|
print("Fetching files")
|
||||||
|
fetched_files, fetch_error = mlq.Fetch(test_directory)
|
||||||
|
|
||||||
|
if len(fetched_files) < 0:
|
||||||
|
raise Exception("No resource files fetched")
|
||||||
|
else:
|
||||||
|
print("Fetched {} files".format(len(fetched_files)))
|
||||||
|
|
||||||
|
if len(fetch_error) > 0:
|
||||||
|
raise Exception("Some resources not fetched")
|
||||||
|
|
||||||
|
print("Generate the self value, overriding existing")
|
||||||
|
oMarti = mlq.NewMartiDefinition()
|
||||||
|
for full_fileName in fetched_files:
|
||||||
|
if os.path.isfile(full_fileName):
|
||||||
|
oResource = mlq.NewMartiLQResource(full_fileName, "", False, True)
|
||||||
|
oMarti["resources"].append(oResource)
|
||||||
|
print("Perform validation test")
|
||||||
|
lqresults, testError = mlq.TestMartiDefinition("../BSBDirectoryHttp.json")
|
||||||
|
|
||||||
|
testfile = open("./test/LoadQualityTest_Http.csv", "w+", newline ="")
|
||||||
|
with testfile:
|
||||||
|
lqwriter = csv.writer(testfile)
|
||||||
|
lqwriter.writerows(lqresults)
|
||||||
|
|
||||||
|
if testError > 0:
|
||||||
|
print("MISMATCH DETECTED")
|
||||||
|
else:
|
||||||
|
print("RECONCILED")
|
||||||
|
|
||||||
|
mlq.CloseLog()
|
||||||
|
|
||||||
|
print("Test completed: SampleFetchHttpBsb.py")
|
||||||
|
|
||||||
|
|
@ -4,11 +4,15 @@ import os
|
||||||
import json
|
import json
|
||||||
import sys
|
import sys
|
||||||
import csv
|
import csv
|
||||||
|
import zipfile
|
||||||
|
|
||||||
|
|
||||||
sys.path.insert(0, "../../../source/python/client")
|
sys.path.insert(0, "../../../source/python/client")
|
||||||
#from source.python.client.martiLQ import martiLQ
|
|
||||||
from martiLQ import *
|
from martiLQ import *
|
||||||
|
|
||||||
|
ftpFetch = True
|
||||||
|
os.environ["MARTILQ_LOGPATH"] = "./test/logs"
|
||||||
|
|
||||||
def ftpList(host, path):
|
def ftpList(host, path):
|
||||||
|
|
||||||
files = []
|
files = []
|
||||||
|
|
@ -58,7 +62,8 @@ for file_name in files:
|
||||||
if file_name.endswith(".csv") | file_name.endswith(".txt"):
|
if file_name.endswith(".csv") | file_name.endswith(".txt"):
|
||||||
file_remote = remote_dir + file_name
|
file_remote = remote_dir + file_name
|
||||||
file_local = "./test/" + file_name
|
file_local = "./test/" + file_name
|
||||||
# ftpPull(remote_host, file_remote, file_local)
|
if ftpFetch:
|
||||||
|
ftpPull(remote_host, file_remote, file_local)
|
||||||
|
|
||||||
print("Creating martiLQ definition")
|
print("Creating martiLQ definition")
|
||||||
mlq = martiLQ()
|
mlq = martiLQ()
|
||||||
|
|
@ -67,16 +72,56 @@ oMarti = mlq.NewMartiDefinition()
|
||||||
for file_name in files:
|
for file_name in files:
|
||||||
if file_name.startswith("BSBDirectory"):
|
if file_name.startswith("BSBDirectory"):
|
||||||
if file_name.endswith(".csv") | file_name.endswith(".txt"):
|
if file_name.endswith(".csv") | file_name.endswith(".txt"):
|
||||||
oResource = mlq.NewMartiResource(os.path.join("./test/", file_name), "", False, True, "./test/logs")
|
oResource = mlq.NewMartiLQResource(os.path.join("./test/", file_name), "", False, True)
|
||||||
oMarti["resources"].append(oResource)
|
oMarti["resources"].append(oResource)
|
||||||
|
|
||||||
mlq.CloseLog()
|
mlq.CloseLog()
|
||||||
print("Save martiLQ definition")
|
|
||||||
jd = json.dumps(oMarti, indent=5)
|
|
||||||
|
|
||||||
|
|
||||||
|
print("Save martiLQ definition")
|
||||||
jsonFile = open("./test/BSBDirectoryPlain.mti", "w")
|
jsonFile = open("./test/BSBDirectoryPlain.mti", "w")
|
||||||
jsonFile.write(jd)
|
jsonFile.write(json.dumps(oMarti, indent=5))
|
||||||
jsonFile.close()
|
jsonFile.close()
|
||||||
|
print("Base sample mti written: BSBDirectoryPlain.mti")
|
||||||
|
|
||||||
|
|
||||||
|
print("Creating martiLQ ZIP file")
|
||||||
|
zipFileName = "BSBDirectory.zip"
|
||||||
|
fileZipCount = 0
|
||||||
|
|
||||||
|
mlq = martiLQ()
|
||||||
|
oMarti = mlq.NewMartiDefinition()
|
||||||
|
with zipfile.ZipFile("./test/" + zipFileName, "w", compression=zipfile.ZIP_DEFLATED) as zipObj:
|
||||||
|
for file_name in files:
|
||||||
|
if file_name.startswith("BSBDirectory"):
|
||||||
|
if file_name.endswith(".csv") | file_name.endswith(".txt"):
|
||||||
|
file_remote = remote_dir + file_name
|
||||||
|
file_local = "./test/" + file_name
|
||||||
|
if ftpFetch:
|
||||||
|
ftpPull(remote_host, file_remote, file_local)
|
||||||
|
zipObj.write(file_local, file_name)
|
||||||
|
fileZipCount = fileZipCount + 1
|
||||||
|
oResource = mlq.NewMartiLQResource(os.path.join("./test/", file_name), "", False, True)
|
||||||
|
oResource["url"] = "@"+zipFileName + "/" + file_name
|
||||||
|
oMarti["resources"].append(oResource)
|
||||||
|
|
||||||
|
|
||||||
|
oResource = mlq.NewMartiLQResource(os.path.join("./test/", zipFileName), "", False, True)
|
||||||
|
oResource["url"] = "./test/" + zipFileName
|
||||||
|
mlq.SetAttributeValueString(Attributes=oResource["attributes"], Key="compression", Category="format", Function="algo", Value="WINZIP")
|
||||||
|
mlq.SetAttributeValueNumber(Attributes=oResource["attributes"], Key="files", Category="dataset", Function="count", Value=fileZipCount)
|
||||||
|
oMarti["resources"].append(oResource)
|
||||||
|
|
||||||
|
mlq.CloseLog()
|
||||||
|
|
||||||
|
print("Save martiLQ ZIP definition")
|
||||||
|
jsonFile = open("./test/MartiLQ_BSBZip.mti", "w")
|
||||||
|
jsonFile.write(json.dumps(oMarti, indent=5))
|
||||||
|
jsonFile.close()
|
||||||
|
print("ZIP sample mti written: MartiLQ_BSBZip.mti")
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
print("Sample completed: SampleGenerateBsb.py")
|
print("Sample completed: SampleGenerateBsb.py")
|
||||||
|
|
||||||
lqresults, testError = mlq.TestMartiDefinition("./test/BSBDirectoryPlain.mti")
|
lqresults, testError = mlq.TestMartiDefinition("./test/BSBDirectoryPlain.mti")
|
||||||
|
|
|
||||||
|
|
@ -0,0 +1,92 @@
|
||||||
|
|
||||||
|
import ftplib
|
||||||
|
import os
|
||||||
|
import json
|
||||||
|
import sys
|
||||||
|
import csv
|
||||||
|
|
||||||
|
sys.path.insert(0, "../../../source/python/client")
|
||||||
|
from martiLQ import *
|
||||||
|
|
||||||
|
def ftpList(host, path):
|
||||||
|
|
||||||
|
files = []
|
||||||
|
with ftplib.FTP(host) as ftp:
|
||||||
|
try:
|
||||||
|
ftp.login()
|
||||||
|
ftp.cwd(path)
|
||||||
|
files = ftp.nlst()
|
||||||
|
except ftplib.all_errors as e:
|
||||||
|
print('FTP error:', e)
|
||||||
|
|
||||||
|
return files
|
||||||
|
|
||||||
|
|
||||||
|
def ftpPull(host, file_remote, file_local):
|
||||||
|
|
||||||
|
with ftplib.FTP(host) as ftp:
|
||||||
|
|
||||||
|
try:
|
||||||
|
ftp.login()
|
||||||
|
|
||||||
|
with open(file_local, 'wb') as fl:
|
||||||
|
res = ftp.retrbinary(f"RETR {file_remote}", fl.write)
|
||||||
|
if not res.startswith('226 Transfer complete'):
|
||||||
|
print('Download failed')
|
||||||
|
if os.path.isfile(file_local):
|
||||||
|
os.remove(file_local)
|
||||||
|
|
||||||
|
except ftplib.all_errors as e:
|
||||||
|
print('FTP error:', e)
|
||||||
|
if os.path.isfile(file_local):
|
||||||
|
os.remove(file_local)
|
||||||
|
|
||||||
|
|
||||||
|
remote_host = "bsb.hostedftp.com"
|
||||||
|
remote_dir = "/~auspaynetftp/BSB/"
|
||||||
|
|
||||||
|
print("Fetch sample file list")
|
||||||
|
files = ftpList(remote_host, remote_dir)
|
||||||
|
|
||||||
|
if not os.path.exists("./test"):
|
||||||
|
os.mkdir("./test")
|
||||||
|
|
||||||
|
print("Fetch sample files")
|
||||||
|
for file_name in files:
|
||||||
|
if file_name.startswith("BSBDirectory"):
|
||||||
|
if file_name.endswith(".csv") | file_name.endswith(".txt"):
|
||||||
|
file_remote = remote_dir + file_name
|
||||||
|
file_local = "./test/" + file_name
|
||||||
|
ftpPull(remote_host, file_remote, file_local)
|
||||||
|
|
||||||
|
print("Creating martiLQ definition")
|
||||||
|
mlq = martiLQ()
|
||||||
|
oMarti = mlq.NewMartiDefinition()
|
||||||
|
|
||||||
|
for file_name in files:
|
||||||
|
if file_name.startswith("BSBDirectory"):
|
||||||
|
if file_name.endswith(".csv") | file_name.endswith(".txt"):
|
||||||
|
oResource = mlq.NewMartiLQResource(os.path.join("./test/", file_name), "", False, True, "./test/logs")
|
||||||
|
oMarti["resources"].append(oResource)
|
||||||
|
|
||||||
|
mlq.CloseLog()
|
||||||
|
print("Save martiLQ definition")
|
||||||
|
jd = json.dumps(oMarti, indent=5)
|
||||||
|
|
||||||
|
jsonFile = open("./test/BSBDirectoryPlain.mti", "w")
|
||||||
|
jsonFile.write(jd)
|
||||||
|
jsonFile.close()
|
||||||
|
print("Sample completed: SampleGenerateBsb.py")
|
||||||
|
|
||||||
|
lqresults, testError = mlq.TestMartiDefinition("./test/BSBDirectoryPlain.mti")
|
||||||
|
|
||||||
|
testfile = open("./test/LoadQualityTest01.csv", "w+", newline ="")
|
||||||
|
with testfile:
|
||||||
|
lqwriter = csv.writer(testfile)
|
||||||
|
lqwriter.writerows(lqresults)
|
||||||
|
|
||||||
|
if testError:
|
||||||
|
print("MISMATCH DETECTED")
|
||||||
|
|
||||||
|
print("Test completed: SampleGenerateBsb.py")
|
||||||
|
|
||||||
|
|
@ -1,3 +1,12 @@
|
||||||
# Place maker
|
# Java code for martiLQ
|
||||||
|
|
||||||
|
The ``client`` folder contains Java library for creating
|
||||||
|
**martiLQ** definition and comparison for the Load Quality.
|
||||||
|
|
||||||
|
See the Java samples directory on how to use the code.
|
||||||
|
|
||||||
|
__TO COME__
|
||||||
|
|
||||||
|
Publish of Java library to ``Maven`` is yet to occur
|
||||||
|
|
||||||
|
This will occur after merge of code into ``main`` branch
|
||||||
|
|
|
||||||
|
|
@ -144,7 +144,7 @@ Param(
|
||||||
|
|
||||||
Get-ChildItem $SourceFolder -Filter $Filter -Recurse:$Recurse -Force| Where-Object {!$_.PSIsContainer} | ForEach-Object {
|
Get-ChildItem $SourceFolder -Filter $Filter -Recurse:$Recurse -Force| Where-Object {!$_.PSIsContainer} | ForEach-Object {
|
||||||
|
|
||||||
$oResource = New-MartiResource -SourcePath $_.FullName -UrlPath $remoteDirectory -LogPath $LogPath -ExtendAttributes:$ExtendAttributes -ExcludeHash:$ExcludeHash
|
$oResource = New-MartiResource -SourcePath $_.FullName -UrlPath $UrlPath -LogPath $LogPath -ExtendAttributes:$ExtendAttributes -ExcludeHash:$ExcludeHash
|
||||||
|
|
||||||
if ($null -ne $UrlPath -and $UrlPath -ne "") {
|
if ($null -ne $UrlPath -and $UrlPath -ne "") {
|
||||||
$postfixName = $_.FullName.Replace($SourceFullName, "")
|
$postfixName = $_.FullName.Replace($SourceFullName, "")
|
||||||
|
|
@ -289,7 +289,7 @@ function New-DefaultZipAttributes {
|
||||||
$oAttribute = [PSCustomObject]@{
|
$oAttribute = [PSCustomObject]@{
|
||||||
category = "format"
|
category = "format"
|
||||||
name = "compression"
|
name = "compression"
|
||||||
function = "algorithm"
|
function = "algo"
|
||||||
comparison = "NA"
|
comparison = "NA"
|
||||||
value = $CompressionType
|
value = $CompressionType
|
||||||
}
|
}
|
||||||
|
|
@ -298,7 +298,7 @@ function New-DefaultZipAttributes {
|
||||||
$oAttribute = [PSCustomObject]@{
|
$oAttribute = [PSCustomObject]@{
|
||||||
category = "format"
|
category = "format"
|
||||||
name = "encryption"
|
name = "encryption"
|
||||||
function = "algorithm"
|
function = "algo"
|
||||||
comparison = "NA"
|
comparison = "NA"
|
||||||
value = $Encryption
|
value = $Encryption
|
||||||
}
|
}
|
||||||
|
|
|
||||||
|
|
@ -1,3 +1,13 @@
|
||||||
# Place maker
|
# PowerShell code for martiLQ
|
||||||
|
|
||||||
|
This folder contains PowerShell module for creating
|
||||||
|
**martiLQ** definition and comparison for the Load Quality.
|
||||||
|
|
||||||
|
See the PowerShell samples directory on how to use the code.
|
||||||
|
|
||||||
|
__TO COME__
|
||||||
|
|
||||||
|
Publish of PowerShell code as ``Install-Module martiLQ`` is yet to occur
|
||||||
|
|
||||||
|
This will occur after merge of code into ``main`` branch
|
||||||
|
|
||||||
|
|
|
||||||
|
|
@ -1,3 +1,12 @@
|
||||||
# Place maker
|
# Python code for martiLQ
|
||||||
|
|
||||||
|
The ``client`` folder contains Python module for creating
|
||||||
|
**martiLQ** definition and comparison for the Load Quality.
|
||||||
|
|
||||||
|
See the Python samples directory on how to use the code.
|
||||||
|
|
||||||
|
__TO COME__
|
||||||
|
|
||||||
|
Publish of Python code as ``pip install martiLQ`` is yet to occur
|
||||||
|
|
||||||
|
This will occur after merge of code into ``main`` branch
|
||||||
|
|
|
||||||
|
|
@ -8,47 +8,172 @@ import json
|
||||||
import datetime
|
import datetime
|
||||||
import getpass
|
import getpass
|
||||||
import hashlib
|
import hashlib
|
||||||
|
import glob
|
||||||
|
from configparser import ConfigParser
|
||||||
|
import requests
|
||||||
|
import mimetypes
|
||||||
|
|
||||||
class martiLQ:
|
class martiLQ:
|
||||||
|
|
||||||
gLogPathName = ""
|
_SoftwareVersion = "0.0.1"
|
||||||
gSoftwareVersion = "0.0.1"
|
_default_metaFile = "##marti##.mti"
|
||||||
gdefault_metaFile = "##marti##.mti"
|
|
||||||
|
|
||||||
gMartiErrorId = ""
|
_oSoftware = {
|
||||||
gLogOpen = False
|
"extension": "software",
|
||||||
|
"softwareName": "MARTILQREFERENCE",
|
||||||
|
"author": "Meerkat@merebox.com",
|
||||||
|
"version": "0.0.1"
|
||||||
|
}
|
||||||
|
|
||||||
oMartiDefinition = None
|
|
||||||
|
|
||||||
def __init__(self):
|
_MartiErrorId = ""
|
||||||
self.gLogOpen = False
|
_LogOpen = False
|
||||||
|
|
||||||
def Get(self):
|
_oMartiDefinition = None
|
||||||
return self.oMartiDefinition
|
|
||||||
|
_oConfiguration = None
|
||||||
|
|
||||||
def Close(self):
|
|
||||||
if self.gLogOpen:
|
|
||||||
self.CloseLog()
|
|
||||||
self.gLogOpen = False
|
|
||||||
|
|
||||||
def GetSoftwareName(self):
|
def GetSoftwareName(self):
|
||||||
return "MARTILQREFERENCE"
|
return "MARTILQREFERENCE"
|
||||||
|
|
||||||
|
def __init__(self):
|
||||||
|
self._LogOpen = False
|
||||||
|
|
||||||
|
_oSoftware = {
|
||||||
|
"extension": "software",
|
||||||
|
"softwareName": self.GetSoftwareName(),
|
||||||
|
"author": "Meerkat@merebox.com",
|
||||||
|
"version": self._SoftwareVersion
|
||||||
|
}
|
||||||
|
|
||||||
|
self._oConfiguration = {
|
||||||
|
"softwareName": self.GetSoftwareName(),
|
||||||
|
"author": "Meerkat@merebox.com",
|
||||||
|
"version": self._SoftwareVersion,
|
||||||
|
|
||||||
|
"logPath": None,
|
||||||
|
|
||||||
|
"state": "active",
|
||||||
|
"accessLevel": "Confidential",
|
||||||
|
"rights": "Restricted",
|
||||||
|
|
||||||
|
"hashAlgorithm": "SHA256",
|
||||||
|
"signKey_File": None,
|
||||||
|
"signKey_Password": None,
|
||||||
|
|
||||||
|
"loaded": False
|
||||||
|
}
|
||||||
|
|
||||||
|
def LoadConfig(self, ConfigPath=None):
|
||||||
|
|
||||||
|
config_object = ConfigParser()
|
||||||
|
if not ConfigPath is None:
|
||||||
|
if os.path.exists(ConfigPath):
|
||||||
|
config_object.read(ConfigPath)
|
||||||
|
else:
|
||||||
|
self.WriteLog("Configuration path '{}' does not exist".format(ConfigPath))
|
||||||
|
raise Exception("Configuration path '{}' does not exist".format(ConfigPath))
|
||||||
|
else:
|
||||||
|
# Look in default location and name
|
||||||
|
home = os.path.expanduser('~')
|
||||||
|
if os.path.exists(os.path.join(home, ".martilq/martilq.ini")):
|
||||||
|
ConfigPath = os.path.join(home, ".martilq/martilq.ini")
|
||||||
|
if os.path.exists("martilq.ini"):
|
||||||
|
ConfigPath = "martilq.ini"
|
||||||
|
if not ConfigPath is None:
|
||||||
|
self.WriteLog("Usig configuration path '{}'".format(ConfigPath))
|
||||||
|
config_object.read(ConfigPath)
|
||||||
|
|
||||||
|
if config_object.has_section("Resources"):
|
||||||
|
items = config_object["Resource"]
|
||||||
|
if not items is None:
|
||||||
|
config_attr = ["accessLevel", "rights", "state"]
|
||||||
|
for x in config_attr:
|
||||||
|
if not items[x] is None and not items[x] == "":
|
||||||
|
self._oConfiguration[x] = items[x]
|
||||||
|
|
||||||
|
if config_object.has_section("Hash"):
|
||||||
|
items = config_object["Hash"]
|
||||||
|
if not items is None:
|
||||||
|
config_attr = ["hashAlgorithm", "signKey_File", "signKey_Password"]
|
||||||
|
for x in config_attr:
|
||||||
|
if not items[x] is None and not items[x] == "":
|
||||||
|
self._oConfiguration[x] = items[x]
|
||||||
|
|
||||||
|
# Now check environmental values
|
||||||
|
self._oConfiguration["signKey_File"] = os.getenv("MARTILQ_SIGNKEY_FILE", self._oConfiguration["signKey_File"])
|
||||||
|
self._oConfiguration["signKey_Password"] = os.getenv("MARTILQ_SIGNKEY_PASSWORD", self._oConfiguration["signKey_Password"])
|
||||||
|
self._oConfiguration["logPath"] = os.getenv("MARTILQ_LOGPATH", self._oConfiguration["logPath"])
|
||||||
|
|
||||||
|
self.WriteLog("Configuration load processed")
|
||||||
|
|
||||||
|
def Set(self, MartiLQ):
|
||||||
|
self._oMartiDefinition = MartiLQ
|
||||||
|
|
||||||
|
def Get(self):
|
||||||
|
return self._oMartiDefinition
|
||||||
|
|
||||||
|
def Save(self, JsonPath):
|
||||||
|
jsonFile = open(JsonPath, "w")
|
||||||
|
jsonFile.write(json.dumps(self._oMartiDefinition, indent=5))
|
||||||
|
jsonFile.close()
|
||||||
|
|
||||||
|
def Load(self, JsonPath):
|
||||||
|
|
||||||
|
self._MartiErrorId = ""
|
||||||
|
|
||||||
|
self.OpenLog()
|
||||||
|
self.WriteLog("Function 'Load' parameters follow")
|
||||||
|
self.WriteLog("Parameter: SourcePath Value: {}".format(JsonPath))
|
||||||
|
self.WriteLog("")
|
||||||
|
|
||||||
|
if os.path.exists(JsonPath):
|
||||||
|
self.WriteLog("Overwriting existing definition")
|
||||||
|
else:
|
||||||
|
if not os.path.exists(os.path.dirname(JsonPath)):
|
||||||
|
self.WriteLog("Parent folder does not exist")
|
||||||
|
raise Exception("Parent folder '{}' does not exist".format(os.path.dirname(JsonPath)))
|
||||||
|
|
||||||
|
if not self._oMartiDefinition is None:
|
||||||
|
self.WriteLog("Existing definition overwritten")
|
||||||
|
|
||||||
|
jsonFile = open(JsonPath, "r")
|
||||||
|
self._oMartiDefinition = json.load(jsonFile)
|
||||||
|
jsonFile.close()
|
||||||
|
|
||||||
|
|
||||||
|
def SetConfig(self, Key=None, Value=None):
|
||||||
|
|
||||||
|
if not Key is None:
|
||||||
|
self._oConfiguration[Key] = Value
|
||||||
|
|
||||||
|
def GetConfig(self, Key=None):
|
||||||
|
|
||||||
|
if not Key is None:
|
||||||
|
return self._oConfiguration[Key]
|
||||||
|
else:
|
||||||
|
return None
|
||||||
|
|
||||||
|
def Close(self):
|
||||||
|
if self._LogOpen:
|
||||||
|
self.CloseLog()
|
||||||
|
self._LogOpen = False
|
||||||
|
|
||||||
def GetLogName(self):
|
def GetLogName(self):
|
||||||
|
|
||||||
today = datetime.datetime.today()
|
today = datetime.datetime.today()
|
||||||
dateToday = today.strftime("%Y-%m-%d")
|
dateToday = today.strftime("%Y-%m-%d")
|
||||||
|
|
||||||
if None == self.gLogPathName or self.gLogPathName == "":
|
if None == self._oConfiguration["logPath"] or self._oConfiguration["logPath"] == "":
|
||||||
return None
|
return None
|
||||||
|
|
||||||
if not os.path.exists(self.gLogPathName):
|
if not os.path.exists(self._oConfiguration["logPath"]):
|
||||||
os.mkdir(self.gLogPathName)
|
os.mkdir(self._oConfiguration["logPath"])
|
||||||
|
|
||||||
logName = self.GetSoftwareName() + "_" + dateToday + ".log"
|
logName = self.GetSoftwareName() + "_" + dateToday + ".log"
|
||||||
|
|
||||||
return os.path.join(self.gLogPathName, logName)
|
return os.path.join(self._oConfiguration["logPath"], logName)
|
||||||
|
|
||||||
|
|
||||||
def WriteLog(self, LogEntry):
|
def WriteLog(self, LogEntry):
|
||||||
|
|
@ -75,55 +200,48 @@ class martiLQ:
|
||||||
|
|
||||||
def OpenLog(self):
|
def OpenLog(self):
|
||||||
|
|
||||||
if not self.gLogOpen:
|
if not self._LogOpen:
|
||||||
today = datetime.datetime.today()
|
today = datetime.datetime.today()
|
||||||
dateToday = today.strftime("%Y-%m-%d")
|
dateToday = today.strftime("%Y-%m-%d")
|
||||||
self.WriteLog("***********************************************************************************")
|
self.WriteLog("***********************************************************************************")
|
||||||
self.WriteLog("* Start of processing: {}".format(dateToday))
|
self.WriteLog("* Start of processing: {}".format(dateToday))
|
||||||
self.WriteLog("***********************************************************************************")
|
self.WriteLog("***********************************************************************************")
|
||||||
self.gLogOpen = True
|
self._LogOpen = True
|
||||||
|
|
||||||
def CloseLog(self):
|
def CloseLog(self):
|
||||||
|
|
||||||
if self.gLogOpen:
|
if self._LogOpen:
|
||||||
today = datetime.datetime.today()
|
today = datetime.datetime.today()
|
||||||
dateToday = today.strftime("%Y-%m-%d")
|
dateToday = today.strftime("%Y-%m-%d")
|
||||||
self.WriteLog("***********************************************************************************")
|
self.WriteLog("***********************************************************************************")
|
||||||
self.WriteLog("* End of processing: {}".format(dateToday))
|
self.WriteLog("* End of processing: {}".format(dateToday))
|
||||||
self.WriteLog("***********************************************************************************")
|
self.WriteLog("***********************************************************************************")
|
||||||
self.gLogOpen = False
|
self._LogOpen = False
|
||||||
|
|
||||||
def NewMartiDefinition(self):
|
def NewMartiDefinition(self):
|
||||||
|
|
||||||
today = datetime.datetime.today()
|
today = datetime.datetime.today()
|
||||||
dateToday = today.strftime("%Y-%m-%d")
|
dateToday = today.strftime("%Y-%m-%d")
|
||||||
|
|
||||||
oSoftware = {
|
|
||||||
"extension": "software",
|
|
||||||
"softwareName": self.GetSoftwareName(),
|
|
||||||
"author": "Meerkat@merebox.com",
|
|
||||||
"version": self.gSoftwareVersion
|
|
||||||
}
|
|
||||||
|
|
||||||
publisher = getpass.getuser()
|
publisher = getpass.getuser()
|
||||||
|
|
||||||
lcustom = []
|
lcustom = []
|
||||||
lcustom.append(oSoftware)
|
lcustom.append(self._oSoftware)
|
||||||
|
|
||||||
lresource = []
|
lresource = []
|
||||||
|
|
||||||
self.oMartiDefinition = {
|
self._oMartiDefinition = {
|
||||||
|
"content-type": "application/vnd.martilq.json",
|
||||||
"title": "",
|
"title": "",
|
||||||
"uid": str(uuid.uuid4()),
|
"uid": str(uuid.uuid4()),
|
||||||
"resources": lresource,
|
|
||||||
|
|
||||||
"description": "",
|
"description": "",
|
||||||
"modified": dateToday,
|
"modified": dateToday,
|
||||||
"tags": ["document", self.GetSoftwareName()],
|
|
||||||
"publisher": publisher,
|
"publisher": publisher,
|
||||||
"contactPoint": "",
|
"contactPoint": "",
|
||||||
"accessLevel": "Confidential",
|
"accessLevel": self.GetConfig("accessLevel"),
|
||||||
"rights": "Restricted",
|
"rights": self.GetConfig("rights"),
|
||||||
|
"tags": [],
|
||||||
"license": "",
|
"license": "",
|
||||||
"state": "active",
|
"state": "active",
|
||||||
"batch": 1.0,
|
"batch": 1.0,
|
||||||
|
|
@ -131,23 +249,54 @@ class martiLQ:
|
||||||
"landingPage": "",
|
"landingPage": "",
|
||||||
"theme": "",
|
"theme": "",
|
||||||
|
|
||||||
|
"resources": lresource,
|
||||||
"custom": lcustom
|
"custom": lcustom
|
||||||
}
|
}
|
||||||
|
|
||||||
return self.oMartiDefinition
|
return self._oMartiDefinition
|
||||||
|
|
||||||
|
|
||||||
|
def NewMartiChildItem(self, SourceFolder, UrlPath=None, Recurse=True, ExtendAttributes=True, ExcludeHash=False, Filter ="*"):
|
||||||
|
|
||||||
def NewMartiResource(self, SourcePath, UrlPath, ExcludeHash, ExtendAttributes, LogPath):
|
SourceFullName = os.path.abspath(SourceFolder)
|
||||||
|
|
||||||
|
for fullName in glob.iglob(SourceFullName, recursive=Recurse):
|
||||||
|
if os.path.isfile(fullName):
|
||||||
|
oResource = self.NewMartiLQResource(SourcePath=fullName, UrlPath=UrlPath, ExtendAttributes=ExtendAttributes, ExcludeHash=ExcludeHash)
|
||||||
|
self._oMartiDefinition["resources"].append(oResource)
|
||||||
|
|
||||||
|
|
||||||
|
def GetContentType(self, SourcePath):
|
||||||
|
|
||||||
|
ext = None
|
||||||
|
|
||||||
|
# Some overrides
|
||||||
|
match = str(os.path.splitext(SourcePath)[1][1:]).lower()
|
||||||
|
if (ext is None or ext == "") and match == "csv":
|
||||||
|
ext = "text/csv"
|
||||||
|
|
||||||
|
if ext is None or ext == "":
|
||||||
|
ext, _ = mimetypes.guess_type(SourcePath, strict=True)
|
||||||
|
|
||||||
|
if ext is None or ext == "":
|
||||||
|
match = str(os.path.splitext(SourcePath)[1][1:]).lower()
|
||||||
|
if match == "md":
|
||||||
|
ext = "application/markdown"
|
||||||
|
|
||||||
|
if ext is None or ext == "":
|
||||||
|
ext = "application/vnd.unknown." + os.path.splitext(SourcePath)[1][1:]
|
||||||
|
|
||||||
|
return ext
|
||||||
|
|
||||||
|
def NewMartiLQResource(self, SourcePath, UrlPath, ExcludeHash, ExtendAttributes):
|
||||||
|
|
||||||
today = datetime.datetime.today()
|
today = datetime.datetime.today()
|
||||||
dateToday = today.strftime("%Y-%m-%d")
|
dateToday = today.strftime("%Y-%m-%dT%H:%M:%S")
|
||||||
|
|
||||||
self.gMartiErrorId = ""
|
self._MartiErrorId = ""
|
||||||
self.gLogPathName = LogPath
|
|
||||||
|
|
||||||
self.OpenLog()
|
self.OpenLog()
|
||||||
self.WriteLog("Function 'NewMartiResource' parameters follow")
|
self.WriteLog("Function 'NewMartiLQResource' parameters follow")
|
||||||
self.WriteLog("Parameter: SourcePath Value: {}".format(SourcePath))
|
self.WriteLog("Parameter: SourcePath Value: {}".format(SourcePath))
|
||||||
self.WriteLog("Parameter: UrlPath Value: {}".format(UrlPath))
|
self.WriteLog("Parameter: UrlPath Value: {}".format(UrlPath))
|
||||||
self.WriteLog("Parameter: ExcludeHash Value: {}".format(ExcludeHash))
|
self.WriteLog("Parameter: ExcludeHash Value: {}".format(ExcludeHash))
|
||||||
|
|
@ -157,20 +306,20 @@ class martiLQ:
|
||||||
|
|
||||||
item = os.path.basename(SourcePath)
|
item = os.path.basename(SourcePath)
|
||||||
self.WriteLog("Define file {}".format(SourcePath))
|
self.WriteLog("Define file {}".format(SourcePath))
|
||||||
HashAlgorithm = "SHA256"
|
HashAlgorithm = self.GetConfig("hashAlgorithm")
|
||||||
|
|
||||||
try:
|
try:
|
||||||
mtime = os.path.getmtime(SourcePath)
|
mtime = os.path.getmtime(SourcePath)
|
||||||
except OSError:
|
except OSError:
|
||||||
mtime = 0
|
mtime = 0
|
||||||
last_modified_date = datetime.datetime.fromtimestamp(mtime).strftime("%Y-%m-%d %H:%M:%S")
|
last_modified_date = datetime.datetime.fromtimestamp(mtime).strftime("%Y-%m-%dT%H:%M:%S")
|
||||||
|
|
||||||
if ExcludeHash:
|
if ExcludeHash:
|
||||||
hash = None
|
hash = None
|
||||||
else:
|
else:
|
||||||
hash = self.NewMartiLQHash(Algorithm=HashAlgorithm, FilePath=SourcePath, Value="")
|
hash = self.NewMartiLQHash(Algorithm=HashAlgorithm, FilePath=SourcePath, Value="", Sign=self.GetConfig("signKey_File"))
|
||||||
|
|
||||||
lattribute = self.SetMartiResourceAttributes(SourcePath, os.path.splitext(SourcePath)[1][1:], ExtendAttributes)
|
lattribute = self.SetMartiLQResourceAttributes(SourcePath, str(os.path.splitext(SourcePath)[1][1:]).lower(), ExtendAttributes)
|
||||||
|
|
||||||
oResource = {
|
oResource = {
|
||||||
"title": item.replace(os.path.splitext(SourcePath)[1], ""),
|
"title": item.replace(os.path.splitext(SourcePath)[1], ""),
|
||||||
|
|
@ -178,15 +327,15 @@ class martiLQ:
|
||||||
"documentName": item,
|
"documentName": item,
|
||||||
"issuedDate": dateToday,
|
"issuedDate": dateToday,
|
||||||
"modified": last_modified_date,
|
"modified": last_modified_date,
|
||||||
"state": "active",
|
"state": self.GetConfig("state"),
|
||||||
"author": "",
|
"author": self.GetConfig("author"),
|
||||||
"length": os.path.getsize(SourcePath),
|
"length": os.path.getsize(SourcePath),
|
||||||
"hash": hash,
|
"hash": hash,
|
||||||
|
|
||||||
"description": "",
|
"description": "",
|
||||||
"url": "",
|
"url": "",
|
||||||
"version": self.gSoftwareVersion,
|
"version": "",
|
||||||
"format": os.path.splitext(SourcePath)[1][1:],
|
"content-type": self.GetContentType(SourcePath),
|
||||||
"compression": None,
|
"compression": None,
|
||||||
"encryption": None,
|
"encryption": None,
|
||||||
|
|
||||||
|
|
@ -194,7 +343,7 @@ class martiLQ:
|
||||||
}
|
}
|
||||||
|
|
||||||
if None != UrlPath and UrlPath != "":
|
if None != UrlPath and UrlPath != "":
|
||||||
if UrlPath[UrlPath.Length-1] == "/" or UrlPath[UrlPath.Length-1] == "\\":
|
if UrlPath[len(UrlPath)-1] == "/" or UrlPath[len(UrlPath)-1] == "\\":
|
||||||
oResource["url"] = UrlPath.replace("\\", "/") + item
|
oResource["url"] = UrlPath.replace("\\", "/") + item
|
||||||
else:
|
else:
|
||||||
oResource["url"] = UrlPath.replace("\\", "/") + "/" + item
|
oResource["url"] = UrlPath.replace("\\", "/") + "/" + item
|
||||||
|
|
@ -202,41 +351,75 @@ class martiLQ:
|
||||||
self.WriteLog("Complete file {}".format(SourcePath))
|
self.WriteLog("Complete file {}".format(SourcePath))
|
||||||
|
|
||||||
else:
|
else:
|
||||||
self.gMartiErrorId = "MRI2001"
|
self._MartiErrorId = "MRI2001"
|
||||||
message = "Document '{}' not found or is a folder".format(SourcePath)
|
message = "Document '{}' not found or is a folder".format(SourcePath)
|
||||||
self.WriteLog(message + " " + self.gMartiErrorId)
|
self.WriteLog(message + " " + self._MartiErrorId)
|
||||||
raise Exception(message)
|
raise Exception(message)
|
||||||
|
|
||||||
return oResource
|
return oResource
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
def NewMartiLQHash(self, Algorithm, FilePath, Value=""):
|
def NewMartiLQHash(self, Algorithm, FilePath, Value="", Sign=""):
|
||||||
|
|
||||||
if Value == "" and FilePath != "":
|
try:
|
||||||
if Algorithm == "SHA256":
|
signed = False
|
||||||
sha_hash = hashlib.sha256()
|
if Value == "" and FilePath != "":
|
||||||
with open(FilePath,"rb") as fh:
|
|
||||||
for byte_block in iter(lambda: fh.read(4096),b""):
|
|
||||||
sha_hash.update(byte_block)
|
|
||||||
Value = sha_hash.hexdigest()
|
|
||||||
if Algorithm == "SHA512":
|
|
||||||
sha_hash = hashlib.sha512()
|
|
||||||
with open(FilePath,"rb") as fh:
|
|
||||||
for byte_block in iter(lambda: fh.read(4096),b""):
|
|
||||||
sha_hash.update(byte_block)
|
|
||||||
Value = sha_hash.hexdigest()
|
|
||||||
if Algorithm == "MD5":
|
|
||||||
sha_hash = hashlib.md5()
|
|
||||||
with open(FilePath,"rb") as fh:
|
|
||||||
for byte_block in iter(lambda: fh.read(4096),b""):
|
|
||||||
sha_hash.update(byte_block)
|
|
||||||
Value = sha_hash.hexdigest()
|
|
||||||
|
|
||||||
oHash = {
|
if not Sign is None and not Sign == "" and not os.path.exists(Sign):
|
||||||
"algo": Algorithm,
|
self.WriteLog("Sign file '{}' not found".format(Sign))
|
||||||
"value": Value
|
Sign = ""
|
||||||
}
|
|
||||||
|
if Sign is None or Sign == "":
|
||||||
|
|
||||||
|
if Algorithm == "SHA256":
|
||||||
|
sha_hash = hashlib.sha256()
|
||||||
|
with open(FilePath,"rb") as fh:
|
||||||
|
for byte_block in iter(lambda: fh.read(4096),b""):
|
||||||
|
sha_hash.update(byte_block)
|
||||||
|
Value = sha_hash.hexdigest()
|
||||||
|
if Algorithm == "SHA512":
|
||||||
|
sha_hash = hashlib.sha512()
|
||||||
|
with open(FilePath,"rb") as fh:
|
||||||
|
for byte_block in iter(lambda: fh.read(4096),b""):
|
||||||
|
sha_hash.update(byte_block)
|
||||||
|
Value = sha_hash.hexdigest()
|
||||||
|
if Algorithm == "MD5":
|
||||||
|
sha_hash = hashlib.md5()
|
||||||
|
with open(FilePath,"rb") as fh:
|
||||||
|
for byte_block in iter(lambda: fh.read(4096),b""):
|
||||||
|
sha_hash.update(byte_block)
|
||||||
|
Value = sha_hash.hexdigest()
|
||||||
|
|
||||||
|
else:
|
||||||
|
|
||||||
|
import OpenSSL
|
||||||
|
from OpenSSL import crypto
|
||||||
|
import base64
|
||||||
|
|
||||||
|
private_key_file = open(Sign, "r")
|
||||||
|
privkey = private_key_file.read()
|
||||||
|
private_key_file.close()
|
||||||
|
password = self.GetConfig("sigenKey_Password")
|
||||||
|
|
||||||
|
if privkey.startswith('-----BEGIN '):
|
||||||
|
pkey = crypto.load_privatekey(crypto.FILETYPE_PEM, privkey, password.encode('utf-8'))
|
||||||
|
else:
|
||||||
|
pkey = crypto.load_pkcs12(privkey, password).get_privatekey()
|
||||||
|
with open(FilePath,"rb") as fh:
|
||||||
|
sign = OpenSSL.crypto.sign(pkey, fh.read(), Algorithm)
|
||||||
|
Value = (base64.b64encode(sign)).decode('utf-8')
|
||||||
|
signed = True
|
||||||
|
|
||||||
|
oHash = {
|
||||||
|
"algo": Algorithm,
|
||||||
|
"value": Value,
|
||||||
|
"signed": signed
|
||||||
|
}
|
||||||
|
|
||||||
|
except Exception as e:
|
||||||
|
self.WriteLog("Hash error for file {}: {}".format(FilePath, str(e)))
|
||||||
|
raise e
|
||||||
|
|
||||||
return oHash
|
return oHash
|
||||||
|
|
||||||
|
|
@ -251,10 +434,6 @@ class martiLQ:
|
||||||
return oEncryption
|
return oEncryption
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
def SetMartiAttribute(self, Attributes, ACategory, AName, AFunction, Comparison, Value):
|
def SetMartiAttribute(self, Attributes, ACategory, AName, AFunction, Comparison, Value):
|
||||||
|
|
||||||
matched = False
|
matched = False
|
||||||
|
|
@ -396,7 +575,7 @@ class martiLQ:
|
||||||
oAttribute = {
|
oAttribute = {
|
||||||
"category": "format",
|
"category": "format",
|
||||||
"name": "compression",
|
"name": "compression",
|
||||||
"function": "algorithm",
|
"function": "algo",
|
||||||
"comparison": "NA",
|
"comparison": "NA",
|
||||||
"value": CompressionType
|
"value": CompressionType
|
||||||
}
|
}
|
||||||
|
|
@ -405,7 +584,7 @@ class martiLQ:
|
||||||
oAttribute = {
|
oAttribute = {
|
||||||
"category": "format",
|
"category": "format",
|
||||||
"name": "encryption",
|
"name": "encryption",
|
||||||
"function": "algorithm",
|
"function": "algo",
|
||||||
"comparison": "NA",
|
"comparison": "NA",
|
||||||
"value": Encryption
|
"value": Encryption
|
||||||
}
|
}
|
||||||
|
|
@ -423,7 +602,7 @@ class martiLQ:
|
||||||
return lattribute
|
return lattribute
|
||||||
|
|
||||||
|
|
||||||
def SetAttributeValueString(self, Attributes,Category,Key,Function,Value,Comparison = "EQ"):
|
def SetAttributeValueString(self, Attributes, Category, Key, Function, Value, Comparison="EQ"):
|
||||||
|
|
||||||
for item in Attributes:
|
for item in Attributes:
|
||||||
|
|
||||||
|
|
@ -447,7 +626,7 @@ class martiLQ:
|
||||||
return
|
return
|
||||||
|
|
||||||
|
|
||||||
def SetAttributeValueNumber(self, Attributes,Category,Key,Function,Value,Comparison = "EQ"):
|
def SetAttributeValueNumber(self, Attributes, Category, Key, Function, Value, Comparison = "EQ"):
|
||||||
|
|
||||||
for item in Attributes:
|
for item in Attributes:
|
||||||
|
|
||||||
|
|
@ -472,7 +651,7 @@ class martiLQ:
|
||||||
return
|
return
|
||||||
|
|
||||||
|
|
||||||
def SetMartiResourceAttributes(self, PathFile, FileType, ExtendedAttributes):
|
def SetMartiLQResourceAttributes(self, PathFile, FileType, ExtendedAttributes):
|
||||||
|
|
||||||
lattribute = None
|
lattribute = None
|
||||||
|
|
||||||
|
|
@ -526,10 +705,12 @@ class martiLQ:
|
||||||
if FileType == "zip":
|
if FileType == "zip":
|
||||||
lattribute = self.NewDefaultZipAttributes("ZIP")
|
lattribute = self.NewDefaultZipAttributes("ZIP")
|
||||||
if ExtendedAttributes:
|
if ExtendedAttributes:
|
||||||
self.SetAttributeValueNumber(lattribute, "dataset", "files", "count", -1)
|
self.SetAttributeValueNumber(lattribute, "dataset", "files", "count", 0, Comparison="NA")
|
||||||
|
|
||||||
if FileType == "7z":
|
if FileType == "7z":
|
||||||
lattribute = self.NewDefaultZipAttributes("7Z")
|
lattribute = self.NewDefaultZipAttributes("7Z")
|
||||||
|
if ExtendedAttributes:
|
||||||
|
self.SetAttributeValueNumber(lattribute, "dataset", "files", "count", 0, Comparison="NA")
|
||||||
|
|
||||||
if lattribute == None:
|
if lattribute == None:
|
||||||
lattribute = []
|
lattribute = []
|
||||||
|
|
@ -537,6 +718,81 @@ class martiLQ:
|
||||||
|
|
||||||
return lattribute
|
return lattribute
|
||||||
|
|
||||||
|
def FtpPull(self, host, file_remote, file_local):
|
||||||
|
|
||||||
|
with ftplib.FTP(host) as ftp:
|
||||||
|
|
||||||
|
try:
|
||||||
|
ftp.login()
|
||||||
|
|
||||||
|
with open(file_local, 'wb') as fl:
|
||||||
|
res = ftp.retrbinary(f"RETR {file_remote}", fl.write)
|
||||||
|
if not res.startswith('226 Transfer complete'):
|
||||||
|
print('Download failed')
|
||||||
|
if os.path.isfile(file_local):
|
||||||
|
os.remove(file_local)
|
||||||
|
|
||||||
|
except ftplib.all_errors as e:
|
||||||
|
print('FTP error:', e)
|
||||||
|
if os.path.isfile(file_local):
|
||||||
|
os.remove(file_local)
|
||||||
|
|
||||||
|
|
||||||
|
def Fetch(self, TargetPath):
|
||||||
|
|
||||||
|
if TargetPath is None or TargetPath == "":
|
||||||
|
self.WriteLog("Target path is missing from fetch")
|
||||||
|
raise Exception("Target path is missing from fetch")
|
||||||
|
|
||||||
|
if self._oMartiDefinition is None:
|
||||||
|
self.WriteLog("No defintion loaded")
|
||||||
|
raise Exception("No defintion loaded")
|
||||||
|
|
||||||
|
if len(self._oMartiDefinition["resources"]) < 1:
|
||||||
|
self.WriteLog("No resources in defintion")
|
||||||
|
raise Exception("No resources in defintion")
|
||||||
|
|
||||||
|
fetched_files = []
|
||||||
|
fetch_error = []
|
||||||
|
|
||||||
|
for resource in self._oMartiDefinition["resources"]:
|
||||||
|
|
||||||
|
if not resource["url"] is None and not resource["url"] == "":
|
||||||
|
method = str(resource["url"].split(":", 2)[0]).lower()
|
||||||
|
#print("Method of fetch {} for {}".format(method, resource["url"]))
|
||||||
|
matched = False
|
||||||
|
if method == "ftp":
|
||||||
|
matched = True
|
||||||
|
parts = resource["url"].split("/", 3)
|
||||||
|
host = parts[2]
|
||||||
|
file_remote = parts[3]
|
||||||
|
self.FtpPull(host, file_remote, os.path.join(TargetPath, resource["documentName"]))
|
||||||
|
fetched_files.append(os.path.join(TargetPath, resource["documentName"]))
|
||||||
|
|
||||||
|
if method == "http" or method == "https":
|
||||||
|
matched = True
|
||||||
|
response = requests.get(resource["url"])
|
||||||
|
if not response.status_code == 200:
|
||||||
|
self.WriteLog("HTP fetch failed with code {} for '{}'".format(response.status_code, resource["url"]))
|
||||||
|
print("HTP fetch failed with code {} for '{}'".format(response.status_code, resource["url"]))
|
||||||
|
fetch_error.append(resource["url"])
|
||||||
|
else:
|
||||||
|
with open(os.path.join(TargetPath, resource["documentName"]),'wb') as fh:
|
||||||
|
fh.write(response.content)
|
||||||
|
fetched_files.append(os.path.join(TargetPath, resource["documentName"]))
|
||||||
|
|
||||||
|
if method == "file":
|
||||||
|
pass
|
||||||
|
|
||||||
|
if not matched:
|
||||||
|
fetch_error.append(resource["documentName"])
|
||||||
|
|
||||||
|
else:
|
||||||
|
fetch_error.append(resource["documentName"])
|
||||||
|
|
||||||
|
return fetched_files, fetch_error
|
||||||
|
|
||||||
|
|
||||||
def TestAttributeDefinition(self, oTestResults, documentName, localR, remoteR):
|
def TestAttributeDefinition(self, oTestResults, documentName, localR, remoteR):
|
||||||
|
|
||||||
errorCount = 0
|
errorCount = 0
|
||||||
|
|
@ -564,57 +820,88 @@ class martiLQ:
|
||||||
|
|
||||||
return errorCount
|
return errorCount
|
||||||
|
|
||||||
def TestMartiDefinition(self, LQSourcePath, oMarti = None, LogPath =""):
|
def TestMartiDefinition(self, SourcePath, Sign=None):
|
||||||
|
|
||||||
self.gMartiErrorId = ""
|
self._MartiErrorId = ""
|
||||||
self.gLogPathName = LogPath
|
|
||||||
|
|
||||||
self.OpenLog()
|
self.OpenLog()
|
||||||
self.WriteLog("Function 'TestMartiDefinition' parameters follow")
|
self.WriteLog("Function 'TestMartiDefinition' parameters follow")
|
||||||
self.WriteLog("Parameter: oMarti Value: {}".format(oMarti))
|
self.WriteLog("Parameter: SourcePath Value: {}".format(SourcePath))
|
||||||
self.WriteLog("Parameter: LQSourcePath Value: {}".format(LQSourcePath))
|
|
||||||
self.WriteLog("")
|
self.WriteLog("")
|
||||||
|
|
||||||
if oMarti is None:
|
if self._oMartiDefinition is None:
|
||||||
oMarti = self.oMartiDefinition
|
|
||||||
|
|
||||||
if oMarti is None:
|
|
||||||
pass
|
pass
|
||||||
|
|
||||||
if not os.path.exists(LQSourcePath):
|
if not os.path.exists(SourcePath):
|
||||||
pass
|
pass
|
||||||
|
|
||||||
jsonFile = open(LQSourcePath, "r")
|
jsonFile = open(SourcePath, "r")
|
||||||
lq = json.load(jsonFile)
|
lq = json.load(jsonFile)
|
||||||
jsonFile.close()
|
jsonFile.close()
|
||||||
|
|
||||||
testError = False
|
testError = 0
|
||||||
oTestResults = []
|
oTestResults = []
|
||||||
|
|
||||||
otest = ["ResourceName", "Level", "Metric", "Matches", "LocalCalculation", "SuppliedValue" ]
|
otest = ["ResourceName", "Level", "Metric", "Matches", "LocalCalculation", "SuppliedValue" ]
|
||||||
oTestResults.append(otest)
|
oTestResults.append(otest)
|
||||||
|
|
||||||
otest = ["", "Batch", "Resource count", (len(oMarti["resources"]) == len(lq["resources"])), len(oMarti["resources"]), len(lq["resources"]) ]
|
otest = ["", "Batch", "Resource count", (len(self._oMartiDefinition["resources"]) == len(lq["resources"])), len(self._oMartiDefinition["resources"]), len(lq["resources"]) ]
|
||||||
oTestResults.append(otest)
|
oTestResults.append(otest)
|
||||||
|
|
||||||
for resource in oMarti["resources"]:
|
for resource in self._oMartiDefinition["resources"]:
|
||||||
|
|
||||||
for retarget in lq["resources"]:
|
for retarget in lq["resources"]:
|
||||||
if resource["documentName"] == retarget["documentName"]:
|
if resource["documentName"] == retarget["documentName"]:
|
||||||
|
|
||||||
testError = testError or resource["hash"]["value"] != retarget["hash"]["value"]
|
if retarget["hash"]["signed"]:
|
||||||
otest = [resource["documentName"], "Resource", "Hash", (resource["hash"]["value"] == retarget["hash"]["value"]), resource["hash"]["value"], retarget["hash"]["value"] ]
|
# Need to verify the hash
|
||||||
oTestResults.append(otest)
|
if Sign is None:
|
||||||
|
Sign = self.GetConfig("signKey_file")
|
||||||
|
|
||||||
testError = testError or resource["length"] != retarget["length"]
|
if Sign is None:
|
||||||
|
self.WriteLog("No Sign Key specified so Hash check cannot be performed for signed content")
|
||||||
|
else:
|
||||||
|
try:
|
||||||
|
import OpenSSL
|
||||||
|
from OpenSSL import crypto
|
||||||
|
import base64
|
||||||
|
except ImportError:
|
||||||
|
self.WriteLog("Import error in signed verification")
|
||||||
|
|
||||||
|
pub_key_file = open(Sign, "r")
|
||||||
|
pubkey = pub_key_file.read()
|
||||||
|
pub_key_file.close()
|
||||||
|
|
||||||
|
x509 = crypto.X509()
|
||||||
|
x509.set_pubkey(pubkey)
|
||||||
|
|
||||||
|
try:
|
||||||
|
crypto.verify(x509, retarget["hash"]["value"], resource["hash"]["value"], retarget["hash"]["algo"])
|
||||||
|
otest = [resource["documentName"], "Resource", "Hash",False, resource["hash"]["value"], retarget["hash"]["value"] ]
|
||||||
|
except:
|
||||||
|
testError = testError + 1
|
||||||
|
self.WriteLog("Error in verification for {}".format(resource["documentName"]))
|
||||||
|
otest = [resource["documentName"], "Resource", "Hash", True, resource["hash"]["value"], retarget["hash"]["value"] ]
|
||||||
|
|
||||||
|
oTestResults.append(otest)
|
||||||
|
|
||||||
|
pass
|
||||||
|
else:
|
||||||
|
if not resource["hash"]["value"] == retarget["hash"]["value"]:
|
||||||
|
testError = testError + 1
|
||||||
|
otest = [resource["documentName"], "Resource", "Hash", (resource["hash"]["value"] == retarget["hash"]["value"]), resource["hash"]["value"], retarget["hash"]["value"] ]
|
||||||
|
oTestResults.append(otest)
|
||||||
|
|
||||||
|
if not resource["length"] == retarget["length"]:
|
||||||
|
testError = testError + 1
|
||||||
otest = [resource["documentName"], "Resource", "Length", (resource["length"] == retarget["length"]), resource["length"], retarget["length"] ]
|
otest = [resource["documentName"], "Resource", "Length", (resource["length"] == retarget["length"]), resource["length"], retarget["length"] ]
|
||||||
oTestResults.append(otest)
|
oTestResults.append(otest)
|
||||||
|
|
||||||
errorAttrCount = self.TestAttributeDefinition(oTestResults, resource["documentName"], resource["attributes"], retarget["attributes"])
|
errorAttrCount = self.TestAttributeDefinition(oTestResults, resource["documentName"], resource["attributes"], retarget["attributes"])
|
||||||
|
testError = testError + errorAttrCount
|
||||||
if errorAttrCount > 0:
|
|
||||||
testError = True
|
|
||||||
|
|
||||||
break
|
break
|
||||||
|
|
||||||
|
self.WriteLog("TestMartiDefinition function completed with {} errors".format(testError))
|
||||||
|
|
||||||
return oTestResults, testError
|
return oTestResults, testError
|
||||||
|
|
|
||||||
|
|
@ -1,44 +0,0 @@
|
||||||
|
|
||||||
. .\source\powershell\MartiLQ.ps1
|
|
||||||
. .\source\powershell\Compress-MartiLQ.ps1
|
|
||||||
|
|
||||||
Write-Host "Test case #1"
|
|
||||||
$oMarti = New-MartiChildItem -SourceFolder ".\docs" -Recurse -UrlPath ".\docs" -Filter "*" -LogPath ".\test\powershell\results\Logs"
|
|
||||||
$oMarti.description = "Sample execution"
|
|
||||||
|
|
||||||
$x = ConvertTo-Json -InputObject $oMarti
|
|
||||||
Set-Content -Path ".\test\powershell\results\marti_test01.mti" -Value $x
|
|
||||||
|
|
||||||
Write-Host "Test case #2"
|
|
||||||
$ArchiveFile = ".\test\powershell\results\marti_test02.zip"
|
|
||||||
Compress-MartiLQ -SourceFolder ".\docs" -Filter "*" -LogPath ".\test\powershell\results\Logs" -ArchiveFile $ArchiveFile
|
|
||||||
|
|
||||||
Write-Host "Test case #3"
|
|
||||||
$y = Get-MartiItem -MartiDefintiion $oMarti -Title "ckan" -Format "txt" -LogPath ".\test\powershell\results\Logs"
|
|
||||||
Write-Host "Get item Title: $($y.title)"
|
|
||||||
Write-Host "Get item Url: $($y.url)"
|
|
||||||
|
|
||||||
Write-Host "Test case #4"
|
|
||||||
$oMarti = New-MartiResource -SourcePath ".\docs\ckan.md" -LogPath ".\test\powershell\results\Logs"
|
|
||||||
$oMarti.description = "Sample execution for ckan"
|
|
||||||
|
|
||||||
$x = ConvertTo-Json -InputObject $oMarti
|
|
||||||
Set-Content -Path ".\test\powershell\results\marti_test02json.mti" -Value $x
|
|
||||||
|
|
||||||
$x = ConvertTo-Csv -InputObject $oMarti
|
|
||||||
Set-Content -Path ".\test\powershell\results\marti_test02csv.mti." -Value $x
|
|
||||||
|
|
||||||
$x = ConvertTo-Xml -As String -InputObject $oMarti -Depth 6
|
|
||||||
Set-Content -Path ".\test\powershell\results\marti_test02xml.mti" -Value $x
|
|
||||||
|
|
||||||
$x = ConvertTo-Html -InputObject $oMarti
|
|
||||||
Set-Content -Path ".\test\powershell\results\marti_test02html.mti" -Value $x
|
|
||||||
|
|
||||||
Write-Host "Test case #5"
|
|
||||||
$oMarti = New-MartiResource -SourcePath ".\docs\eror" -LogPath ".\test\powershell\results\Logs"
|
|
||||||
$oMarti.description = "Sample execution with error"
|
|
||||||
|
|
||||||
$x = ConvertTo-Json -InputObject $oMarti
|
|
||||||
Set-Content -Path ".\test\powershell\results\marti_test03.mti" -Value $x
|
|
||||||
|
|
||||||
|
|
||||||
|
|
@ -1,96 +0,0 @@
|
||||||
|
|
||||||
. .\source\powershell\New-Marti.ps1
|
|
||||||
. .\source\powershell\ConvertFrom-Ckan.ps1
|
|
||||||
|
|
||||||
if (!(Test-Path -Path ".\test\powershell\results\data")) {
|
|
||||||
$null = New-Item -Path ".\test\powershell\results\data" -ItemType Directory
|
|
||||||
}
|
|
||||||
|
|
||||||
#$x = Get-Content -Path ".\test\results\data\bsb.csv"
|
|
||||||
$bsbFile = ".\test\powershell\results\data\bsb.csv"
|
|
||||||
$data = Import-Csv -Path $bsbFile
|
|
||||||
|
|
||||||
$columns = ($data | get-member -type NoteProperty).count
|
|
||||||
$rows = @($data).count
|
|
||||||
|
|
||||||
Write-Host "Rows: $rows Columns: $columns"
|
|
||||||
|
|
||||||
[System.Collections.ArrayList]$lattribute = @()
|
|
||||||
|
|
||||||
$oAttribute = [PSCustomObject]@{
|
|
||||||
category = "dataset"
|
|
||||||
name = "header"
|
|
||||||
function = "count"
|
|
||||||
comparison = "EQ"
|
|
||||||
value = 0
|
|
||||||
}
|
|
||||||
|
|
||||||
$lattribute += $oAttribute
|
|
||||||
|
|
||||||
$oAttribute = [PSCustomObject]@{
|
|
||||||
category = "dataset"
|
|
||||||
name = "footer"
|
|
||||||
function = "count"
|
|
||||||
comparison = "EQ"
|
|
||||||
value = 0
|
|
||||||
}
|
|
||||||
|
|
||||||
$lattribute += $oAttribute
|
|
||||||
|
|
||||||
$oAttribute = [PSCustomObject]@{
|
|
||||||
category = "dataset"
|
|
||||||
name = "rows"
|
|
||||||
function = "count"
|
|
||||||
comparison = "EQ"
|
|
||||||
value = $rows
|
|
||||||
}
|
|
||||||
|
|
||||||
$lattribute += $oAttribute
|
|
||||||
|
|
||||||
$oAttribute = [PSCustomObject]@{
|
|
||||||
category = "dataset"
|
|
||||||
name = "columns"
|
|
||||||
function = "count"
|
|
||||||
comparison = "EQ"
|
|
||||||
value = $columns
|
|
||||||
}
|
|
||||||
|
|
||||||
$lattribute += $oAttribute
|
|
||||||
|
|
||||||
$oAttribute = [PSCustomObject]@{
|
|
||||||
category = "data"
|
|
||||||
name = "BSB"
|
|
||||||
function = "sum"
|
|
||||||
comparison = "EQ"
|
|
||||||
value = 1032092
|
|
||||||
}
|
|
||||||
|
|
||||||
$lattribute += $oAttribute
|
|
||||||
|
|
||||||
$oAttribute = [PSCustomObject]@{
|
|
||||||
category = "data"
|
|
||||||
name = "BSB"
|
|
||||||
function = "unique"
|
|
||||||
comparison = "EQ"
|
|
||||||
value = $rows
|
|
||||||
}
|
|
||||||
|
|
||||||
$lattribute += $oAttribute
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
$uq = Get-Content $bsbFile | ConvertFrom-Csv -Header "C1", "C2" | Select-Object "C2" | Sort-Object "C2" -Unique | Group-Object -Property "C2"
|
|
||||||
$oAttribute = [PSCustomObject]@{
|
|
||||||
category = "data"
|
|
||||||
name = "Institution"
|
|
||||||
function = "unique"
|
|
||||||
comparison = "EQ"
|
|
||||||
value = $uq.Count
|
|
||||||
}
|
|
||||||
|
|
||||||
$lattribute += $oAttribute
|
|
||||||
|
|
||||||
$x = ConvertTo-Json -InputObject $lattribute
|
|
||||||
$x
|
|
||||||
#select -skip 6
|
|
||||||
#| select name
|
|
||||||
|
|
@ -1,41 +0,0 @@
|
||||||
|
|
||||||
. .\source\powershell\New-Marti.ps1
|
|
||||||
. .\source\powershell\ConvertFrom-Ckan.ps1
|
|
||||||
. .\source\powershell\Compare-MartiResource.ps1
|
|
||||||
|
|
||||||
|
|
||||||
$bsbFile = ".\test\powershell\results\data\bsb.csv"
|
|
||||||
|
|
||||||
Write-Host ">>>>>>Test case #1"
|
|
||||||
$x = New-MartiResource -SourcePath $bsbFile -UrlPath "" -ExcludeHash -LogPath ".\test\powershell\results\Logs"
|
|
||||||
|
|
||||||
Write-Host ">>>>>>Test case #2"
|
|
||||||
$x.resources
|
|
||||||
|
|
||||||
[System.Collections.ArrayList] $Attr = Set-MartiAttribute -Attributes $x.resources[0].attributes -ACategory "dataset" -AName "records" -AFunction "count" -comparison "EQ" -value 10
|
|
||||||
$x.resources[0].attributes = Set-MartiAttribute -Attributes $Attr -ACategory "dataset" -AName "columns" -AFunction "count" -comparison "EQ" -value 8
|
|
||||||
|
|
||||||
Write-Host ">>>>>>Test case #3"
|
|
||||||
$x.resources[0].attributes
|
|
||||||
|
|
||||||
Write-Host ">>>>>>Test case #4"
|
|
||||||
$y = Compare-MartiResource -DataSource $bsbFile -Resource $x.resources[0] -LogPath ".\test\powershell\results\Logs"
|
|
||||||
$y
|
|
||||||
|
|
||||||
$covidFile = ".\test\powershell\results\data\covid-case-locations-20210920-1315.json"
|
|
||||||
|
|
||||||
Write-Host ">>>>>>Test case #5"
|
|
||||||
$x = New-MartiResource -SourcePath $covidFile -UrlPath "" -ExcludeHash -LogPath ".\test\powershell\results\Logs"
|
|
||||||
|
|
||||||
Write-Host ">>>>>>Test case #6"
|
|
||||||
$x.resources
|
|
||||||
|
|
||||||
[System.Collections.ArrayList] $Attr = Set-MartiAttribute -Attributes $x.resources[0].attributes -ACategory "dataset" -AName "records" -AFunction "count" -comparison "EQ" -value 10
|
|
||||||
$x.resources[0].attributes = Set-MartiAttribute -Attributes $Attr -ACategory "dataset" -AName "columns" -AFunction "count" -comparison "EQ" -value 8
|
|
||||||
|
|
||||||
Write-Host ">>>>>>Test case #7"
|
|
||||||
$x.resources[0].attributes
|
|
||||||
|
|
||||||
Write-Host ">>>>>>Test case #8"
|
|
||||||
$y = Compare-MartiResource -DataSource $covidFile -Resource $x.resources[0] -LogPath ".\test\powershell\results\Logs"
|
|
||||||
$y
|
|
||||||
|
|
@ -1,18 +0,0 @@
|
||||||
|
|
||||||
. .\source\powershell\New-Marti.ps1
|
|
||||||
. .\source\powershell\ConvertFrom-Ckan.ps1
|
|
||||||
. .\source\powershell\Compare-MartiResource.ps1
|
|
||||||
|
|
||||||
|
|
||||||
$covidFile = ".\test\powershell\results\data\covid-case-locations-20210920-1315.json"
|
|
||||||
|
|
||||||
$x = New-MartiResource -SourcePath $covidFile -UrlPath "" -ExcludeHash -LogPath ".\test\powershell\results\Logs"
|
|
||||||
|
|
||||||
[System.Collections.ArrayList] $attr = Set-MartiAttribute -Attributes $x.resources[0].attributes -ACategory "dataset" -AName "records" -AFunction "count" -comparison "EQ" -value 516
|
|
||||||
$attr = Set-MartiAttribute -Attributes $attr -ACategory "dataset" -AName "columns" -AFunction "count" -comparison "EQ" -value 12
|
|
||||||
$x.resources[0].attributes = Set-MartiAttribute -Attributes $attr -ACategory "format" -AName "list" -AFunction "offset" -comparison "EQ" -value "data.monitor"
|
|
||||||
|
|
||||||
$y = Compare-MartiResource -DataSource $covidFile -Resource $x.resources[0] -LogPath ".\test\powershell\results\Logs"
|
|
||||||
$y
|
|
||||||
|
|
||||||
$attr | Get-Member
|
|
||||||
|
|
@ -0,0 +1,49 @@
|
||||||
|
|
||||||
|
. .\source\powershell\MartiLQ.ps1
|
||||||
|
. .\source\powershell\Compress-MartiLQ.ps1
|
||||||
|
|
||||||
|
try {
|
||||||
|
|
||||||
|
Write-Host "Test case #1"
|
||||||
|
$oMarti = New-MartiChildItem -SourceFolder ".\docs" -Recurse -UrlPath ".\docs" -Filter "*" -LogPath ".\test\powershell\results\Logs"
|
||||||
|
$oMarti.description = "Sample execution"
|
||||||
|
|
||||||
|
$x = ConvertTo-Json -InputObject $oMarti
|
||||||
|
Set-Content -Path ".\test\powershell\results\marti_test01.mti" -Value $x
|
||||||
|
|
||||||
|
Write-Host "Test case #2"
|
||||||
|
$ArchiveFile = ".\test\powershell\results\marti_test02.zip"
|
||||||
|
Compress-MartiLQ -SourceFolder ".\docs" -Filter "*" -LogPath ".\test\powershell\results\Logs" -ArchiveFile $ArchiveFile
|
||||||
|
|
||||||
|
Write-Host "Test case #3"
|
||||||
|
$y = Get-MartiItem -MartiDefintiion $oMarti -Title "ckan" -Format "txt" -LogPath ".\test\powershell\results\Logs"
|
||||||
|
Write-Host "Get item Title: $($y.title)"
|
||||||
|
Write-Host "Get item Url: $($y.url)"
|
||||||
|
|
||||||
|
Write-Host "Test case #4"
|
||||||
|
$oMarti = New-MartiResource -SourcePath ".\docs\ckan.md" -LogPath ".\test\powershell\results\Logs"
|
||||||
|
$oMarti.description = "Sample execution for ckan"
|
||||||
|
|
||||||
|
$x = ConvertTo-Json -InputObject $oMarti
|
||||||
|
Set-Content -Path ".\test\powershell\results\marti_test02json.mti" -Value $x
|
||||||
|
|
||||||
|
$x = ConvertTo-Csv -InputObject $oMarti
|
||||||
|
Set-Content -Path ".\test\powershell\results\marti_test02csv.mti." -Value $x
|
||||||
|
|
||||||
|
$x = ConvertTo-Xml -As String -InputObject $oMarti -Depth 6
|
||||||
|
Set-Content -Path ".\test\powershell\results\marti_test02xml.mti" -Value $x
|
||||||
|
|
||||||
|
$x = ConvertTo-Html -InputObject $oMarti
|
||||||
|
Set-Content -Path ".\test\powershell\results\marti_test02html.mti" -Value $x
|
||||||
|
|
||||||
|
Write-Host "Test case #5"
|
||||||
|
$oMarti = New-MartiResource -SourcePath ".\docs\eror" -LogPath ".\test\powershell\results\Logs"
|
||||||
|
$oMarti.description = "Sample execution with error"
|
||||||
|
|
||||||
|
$x = ConvertTo-Json -InputObject $oMarti
|
||||||
|
Set-Content -Path ".\test\powershell\results\marti_test03.mti" -Value $x
|
||||||
|
|
||||||
|
}
|
||||||
|
catch {
|
||||||
|
throw
|
||||||
|
}
|
||||||
|
|
@ -1,6 +1,6 @@
|
||||||
|
|
||||||
|
|
||||||
. .\source\powershell\New-Marti.ps1
|
. .\source\powershell\MartiLQ.ps1
|
||||||
. .\source\powershell\ConvertFrom-Ckan.ps1
|
. .\source\powershell\ConvertFrom-Ckan.ps1
|
||||||
|
|
||||||
|
|
||||||
|
|
@ -23,7 +23,6 @@ Set-Content -Path ".\test\powershell\results\marti_test06.mti" -Value $x
|
||||||
|
|
||||||
# cases
|
# cases
|
||||||
$covid19 = "https://data.nsw.gov.au/data/api/3/action/package_show?id=3dc5dc39-40b4-4ee9-8ec6-2d862a916dcf"
|
$covid19 = "https://data.nsw.gov.au/data/api/3/action/package_show?id=3dc5dc39-40b4-4ee9-8ec6-2d862a916dcf"
|
||||||
#Invoke-WebRequest $covid19 -Method GET -OutFile ".\test\powershell\results\data\nsw_covid19.csv"
|
|
||||||
$covid_2 = Invoke-WebRequest $covid19
|
$covid_2 = Invoke-WebRequest $covid19
|
||||||
$oMarti = ConvertFrom-Ckan -InputObject $covid_2
|
$oMarti = ConvertFrom-Ckan -InputObject $covid_2
|
||||||
$oMarti.description = "This data has been converted from NSW CKAN data source with URL '$covid19'"
|
$oMarti.description = "This data has been converted from NSW CKAN data source with URL '$covid19'"
|
||||||
|
|
@ -0,0 +1,100 @@
|
||||||
|
|
||||||
|
. .\source\powershell\MartiLQ.ps1
|
||||||
|
. .\source\powershell\ConvertFrom-Ckan.ps1
|
||||||
|
|
||||||
|
if (!(Test-Path -Path ".\test\powershell\results\data")) {
|
||||||
|
$null = New-Item -Path ".\test\powershell\results\data" -ItemType Directory
|
||||||
|
}
|
||||||
|
|
||||||
|
try {
|
||||||
|
|
||||||
|
$bsbFile = ".\test\powershell\results\data\BSBDirectorySep21-306.csv"
|
||||||
|
$data = Import-Csv -Path $bsbFile
|
||||||
|
|
||||||
|
$columns = ($data | get-member -type NoteProperty).count
|
||||||
|
$rows = @($data).count
|
||||||
|
|
||||||
|
Write-Host "Rows: $rows Columns: $columns"
|
||||||
|
|
||||||
|
[System.Collections.ArrayList]$lattribute = @()
|
||||||
|
|
||||||
|
$oAttribute = [PSCustomObject]@{
|
||||||
|
category = "dataset"
|
||||||
|
name = "header"
|
||||||
|
function = "count"
|
||||||
|
comparison = "EQ"
|
||||||
|
value = 0
|
||||||
|
}
|
||||||
|
|
||||||
|
$lattribute += $oAttribute
|
||||||
|
|
||||||
|
$oAttribute = [PSCustomObject]@{
|
||||||
|
category = "dataset"
|
||||||
|
name = "footer"
|
||||||
|
function = "count"
|
||||||
|
comparison = "EQ"
|
||||||
|
value = 0
|
||||||
|
}
|
||||||
|
|
||||||
|
$lattribute += $oAttribute
|
||||||
|
|
||||||
|
$oAttribute = [PSCustomObject]@{
|
||||||
|
category = "dataset"
|
||||||
|
name = "rows"
|
||||||
|
function = "count"
|
||||||
|
comparison = "EQ"
|
||||||
|
value = $rows
|
||||||
|
}
|
||||||
|
|
||||||
|
$lattribute += $oAttribute
|
||||||
|
|
||||||
|
$oAttribute = [PSCustomObject]@{
|
||||||
|
category = "dataset"
|
||||||
|
name = "columns"
|
||||||
|
function = "count"
|
||||||
|
comparison = "EQ"
|
||||||
|
value = $columns
|
||||||
|
}
|
||||||
|
|
||||||
|
$lattribute += $oAttribute
|
||||||
|
|
||||||
|
$oAttribute = [PSCustomObject]@{
|
||||||
|
category = "data"
|
||||||
|
name = "BSB"
|
||||||
|
function = "sum"
|
||||||
|
comparison = "EQ"
|
||||||
|
value = 1032092
|
||||||
|
}
|
||||||
|
|
||||||
|
$lattribute += $oAttribute
|
||||||
|
|
||||||
|
$oAttribute = [PSCustomObject]@{
|
||||||
|
category = "data"
|
||||||
|
name = "BSB"
|
||||||
|
function = "unique"
|
||||||
|
comparison = "EQ"
|
||||||
|
value = $rows
|
||||||
|
}
|
||||||
|
|
||||||
|
$lattribute += $oAttribute
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
$uq = Get-Content $bsbFile | ConvertFrom-Csv -Header "C1", "C2" | Select-Object "C2" | Sort-Object "C2" -Unique | Group-Object -Property "C2"
|
||||||
|
$oAttribute = [PSCustomObject]@{
|
||||||
|
category = "data"
|
||||||
|
name = "Institution"
|
||||||
|
function = "unique"
|
||||||
|
comparison = "EQ"
|
||||||
|
value = $uq.Count
|
||||||
|
}
|
||||||
|
|
||||||
|
$lattribute += $oAttribute
|
||||||
|
|
||||||
|
$x = ConvertTo-Json -InputObject $lattribute
|
||||||
|
$x
|
||||||
|
|
||||||
|
}
|
||||||
|
catch {
|
||||||
|
throw
|
||||||
|
}
|
||||||
|
|
@ -0,0 +1,48 @@
|
||||||
|
|
||||||
|
. .\source\powershell\MartiLQ.ps1
|
||||||
|
. .\source\powershell\MartiLQItem.ps1
|
||||||
|
. .\source\powershell\ConvertFrom-Ckan.ps1
|
||||||
|
. .\source\powershell\Compare-MartiLQResource.ps1
|
||||||
|
|
||||||
|
try {
|
||||||
|
|
||||||
|
$bsbFile = ".\test\powershell\results\data\BSBDirectorySep21-306.csv"
|
||||||
|
|
||||||
|
Write-Host ">>>>>>Test case #1"
|
||||||
|
$x = New-MartiResource -SourcePath $bsbFile -UrlPath "" -ExcludeHash -LogPath ".\test\powershell\results\Logs"
|
||||||
|
|
||||||
|
Write-Host ">>>>>>Test case #2"
|
||||||
|
$x
|
||||||
|
|
||||||
|
[System.Collections.ArrayList] $Attr = Set-MartiAttribute -Attributes $x.attributes -ACategory "dataset" -AName "records" -AFunction "count" -comparison "EQ" -value 10
|
||||||
|
$x.attributes = Set-MartiAttribute -Attributes $Attr -ACategory "dataset" -AName "columns" -AFunction "count" -comparison "EQ" -value 8
|
||||||
|
|
||||||
|
Write-Host ">>>>>>Test case #3"
|
||||||
|
$x.attributes
|
||||||
|
|
||||||
|
Write-Host ">>>>>>Test case #4"
|
||||||
|
$y = Compare-MartiResource -DataSource $bsbFile -Resource $x -LogPath ".\test\powershell\results\Logs"
|
||||||
|
$y
|
||||||
|
|
||||||
|
$covidFile = ".\test\powershell\results\data\covid-case-locations-20210920-1315.json"
|
||||||
|
|
||||||
|
Write-Host ">>>>>>Test case #5"
|
||||||
|
$x = New-MartiResource -SourcePath $covidFile -UrlPath "" -ExcludeHash -LogPath ".\test\powershell\results\Logs"
|
||||||
|
|
||||||
|
Write-Host ">>>>>>Test case #6"
|
||||||
|
$x
|
||||||
|
|
||||||
|
[System.Collections.ArrayList] $Attr = Set-MartiAttribute -Attributes $x.attributes -ACategory "dataset" -AName "records" -AFunction "count" -comparison "EQ" -value 10
|
||||||
|
$x.attributes = Set-MartiAttribute -Attributes $Attr -ACategory "dataset" -AName "columns" -AFunction "count" -comparison "EQ" -value 8
|
||||||
|
|
||||||
|
Write-Host ">>>>>>Test case #7"
|
||||||
|
$x.attributes
|
||||||
|
|
||||||
|
Write-Host ">>>>>>Test case #8"
|
||||||
|
$y = Compare-MartiResource -DataSource $covidFile -Resource $x -LogPath ".\test\powershell\results\Logs"
|
||||||
|
$y
|
||||||
|
|
||||||
|
}
|
||||||
|
catch {
|
||||||
|
throw
|
||||||
|
}
|
||||||
|
|
@ -0,0 +1,25 @@
|
||||||
|
|
||||||
|
. .\source\powershell\MartiLQ.ps1
|
||||||
|
. .\source\powershell\MartiLQItem.ps1
|
||||||
|
. .\source\powershell\ConvertFrom-Ckan.ps1
|
||||||
|
. .\source\powershell\Compare-MartiLQResource.ps1
|
||||||
|
|
||||||
|
try {
|
||||||
|
|
||||||
|
$covidFile = ".\test\powershell\results\data\covid-case-locations-20210920-1315.json"
|
||||||
|
|
||||||
|
$x = New-MartiResource -SourcePath $covidFile -UrlPath "" -ExcludeHash -LogPath ".\test\powershell\results\Logs"
|
||||||
|
|
||||||
|
[System.Collections.ArrayList] $attr = Set-MartiAttribute -Attributes $x.attributes -ACategory "dataset" -AName "records" -AFunction "count" -comparison "EQ" -value 516
|
||||||
|
$attr = Set-MartiAttribute -Attributes $attr -ACategory "dataset" -AName "columns" -AFunction "count" -comparison "EQ" -value 12
|
||||||
|
$x.attributes = Set-MartiAttribute -Attributes $attr -ACategory "format" -AName "list" -AFunction "offset" -comparison "EQ" -value "data.monitor"
|
||||||
|
|
||||||
|
$y = Compare-MartiResource -DataSource $covidFile -Resource $x -LogPath ".\test\powershell\results\Logs"
|
||||||
|
$y
|
||||||
|
|
||||||
|
$attr | Get-Member
|
||||||
|
|
||||||
|
}
|
||||||
|
catch {
|
||||||
|
throw
|
||||||
|
}
|
||||||
|
|
@ -0,0 +1,30 @@
|
||||||
|
-----BEGIN RSA PRIVATE KEY-----
|
||||||
|
Proc-Type: 4,ENCRYPTED
|
||||||
|
DEK-Info: DES-EDE3-CBC,43D90161F4DF49CC
|
||||||
|
|
||||||
|
RLyZTiOgXqyrj8/ecWIKf2oZviplxQCqfhr9s7VfuvPlQwDVroi91d9AnrJvOCFL
|
||||||
|
yyhHTIQxDj0NoEIZ+DDToo2qvoHlmStIpYs1WXvFRZnVsA2KUPmcg9vaRlTb4n4i
|
||||||
|
XU9c/uWDDvdI2k+BuFoeyWaPnn9gBS9y7sPcvBwpDHMMVQ9Quv5q74c9O99Rtb/p
|
||||||
|
gpKWAEO6fLRw8xGyaMnoGCMQbcaa/af6hSuvnFkrJwlkp9fYyfz9bWwwznjwsRBT
|
||||||
|
QC+T5pTp5ZgQ7J/d0Kr21RrBFUlUDtw7C3y5UCY54ajAPuHpoBEXutOGNbzuHNFq
|
||||||
|
sA1i9iu7L2TBT2gp4XlCtnq9RDqAirn/heNDlzJWYAi00a4yratmQepq4n1M3L1e
|
||||||
|
MMb61PNQckvyeWjwLc1QoxK+spyxd9ZjriEXTv7/UjnNgfWXep3KqR4QNRO6u6xq
|
||||||
|
7Bg/87kY1hpLHNLgVhp6laKe68pvxeO3X0OMKw1HOGI9CxdYi92GYXsNgQULscnp
|
||||||
|
ukusKVPHQOUCQe0YzkVcIjrmJ3/EMRvJ3cXRvxmZfal2mJNOsuBwEQunkJRwDBKR
|
||||||
|
WpBArhNRToGG6gCe4+i9vXzyXjYg+szs8XyONY0AzTW+8HXEuBddp+otke7fSsYp
|
||||||
|
QIYCoinNjnXyaTuNHIlFl/9/FXacjaBAS+5UVEbYZ80oYPzcqMhbaUl2be9UDxQn
|
||||||
|
ThjnYoUo+zYGrCqvqVLf/uTM0Wgd/Ac3jZlS86vsHmYi2LXjsWRdSuD11w25pB9r
|
||||||
|
hSrt2n6hRavB8XxOoni6K0lBKTOI4KyxVFDFD2e5aUj2pYpOkG+sjjGM/YQw16Uo
|
||||||
|
yA1IiiOUaBGWZ/EnFhHLD9oeOyQ5LD3F6A2IHSfEYK0JSZhlZKzL1GGrlrgGtY3O
|
||||||
|
gYKijUhjZpGRqlTKO7jG3v6HzQia4sw1mi0C8IlNXZIGqa6HcaCidi3CsFdcKoiB
|
||||||
|
bye0xIIQJPAMR0ybi6ptRAKjINGGlIc1pAfxWq4lu4MDp9zvpo52GaHxvSapaye8
|
||||||
|
JnXDLCfa5gxBBr4WOz7S49gb3UP+6JZUGGKTWsd/RdZNDqmR9GMlTjVx6anMLEwQ
|
||||||
|
YvWyqbPfuzHXe6+XCDYinBTv35eTbZC28FpwvBXniZ8grM0jxqSAi24KQk6f0Ynx
|
||||||
|
KtWwgGsQLy+5BpX1PZAGRnKZBGWOsjOAOgvn1ZWTEzhu8woci6On8ZtgXuLmemws
|
||||||
|
uQdjDCl7B7/fHIElPMg/3FZkYg4cTooazNvBnszbvubkkNFbVFFLn/e3RMSsAuJy
|
||||||
|
FURqYoXp3DkcF1931oJeBjno3MblgQ/6nmWrt/Z6PHd6FVbJqeEEExDWdtz5nWvf
|
||||||
|
BmFK/TFhrn8zeUGbxtZjUhHGTHC9a36ybeFsc04i8yE/8vbpYguEHGBPndro6xpK
|
||||||
|
xIEkZV/EfAgwbu3HiRlVCgLFWyKGz+flG6A1pMeKsbeFmdz4WieuFOjxQtL5Qhek
|
||||||
|
obyPH3XFWv5xFD3TxxMgRKFUXsoWCjAHE5RXi7F0+YJd9GeKqFutohHJyCOFc6L8
|
||||||
|
PZ8b6e7wkh8jqOKwJnlZXDYooI5jvXNWr+KJb96zYWYEHGIkdmvmdg==
|
||||||
|
-----END RSA PRIVATE KEY-----
|
||||||
|
|
@ -0,0 +1,9 @@
|
||||||
|
-----BEGIN PUBLIC KEY-----
|
||||||
|
MIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEAmpZRUYQql7FCIPB67IbS
|
||||||
|
jB15jDRkVARdIrzdnfEx3rXG3l5YI2+oilN8sLaS+oot/PYNQICOHOe1w6hhWnzI
|
||||||
|
eviX2PgA1bf9lNDHYSN3R70F2b1CbrFIA/WXNv58s/yjg5aIjD0lfQ/o1KjAIsUc
|
||||||
|
cA8VKkW8sB0SidjDdHSsyWWF0LKiRGFIAfLgE/spG06zT4M1fB9AgWmqrcCOwW6s
|
||||||
|
DC5dFXnW5ZnaAD5xD8gZdAM/i7JybCOXOJKZLiQNXGLMYp1iFk8fSeuiyJRVIgqI
|
||||||
|
mIYbbPkKaYNBIi9LmUg67HcSODL8uZDtogiRpIZV72NjhsK4iGWvoY9EcKaMNMgS
|
||||||
|
cwIDAQAB
|
||||||
|
-----END PUBLIC KEY-----
|
||||||
|
|
@ -0,0 +1,39 @@
|
||||||
|
|
||||||
|
import os
|
||||||
|
import json
|
||||||
|
import sys
|
||||||
|
import csv
|
||||||
|
import zipfile
|
||||||
|
|
||||||
|
sys.path.insert(0, "./source/python/client")
|
||||||
|
from martiLQ import *
|
||||||
|
|
||||||
|
os.environ["MARTILQ_LOGPATH"] = "./test/python/results/logs"
|
||||||
|
|
||||||
|
print("Python test case #1")
|
||||||
|
|
||||||
|
mlq = martiLQ()
|
||||||
|
mlq.LoadConfig()
|
||||||
|
oMarti = mlq.NewMartiDefinition()
|
||||||
|
mlq.NewMartiChildItem(SourceFolder= "./docs/*", UrlPath="./docs" , ExcludeHash=False, ExtendAttributes=True)
|
||||||
|
|
||||||
|
oMarti["description"] = "Sample execution #1"
|
||||||
|
|
||||||
|
print("Save martiLQ definition #1")
|
||||||
|
mlq.Save("./test/python/results/DocsPlain1.mti")
|
||||||
|
|
||||||
|
print("Save martiLQ definition #2")
|
||||||
|
oMarti["description"] = "Sample execution #2"
|
||||||
|
jsonFile = open("./test/python/results/DocsPlain2.mti", "w")
|
||||||
|
jsonFile.write(json.dumps(oMarti, indent=5))
|
||||||
|
jsonFile.close()
|
||||||
|
print("Base sample mti written: DocsPlain2.mti")
|
||||||
|
|
||||||
|
print("Load martiLQ definition #1")
|
||||||
|
mlq.Load("./test/python/results/DocsPlain1.mti")
|
||||||
|
oMarti = mlq.Get()
|
||||||
|
print("Definition description is: {}".format(oMarti["description"]))
|
||||||
|
|
||||||
|
mlq.CloseLog()
|
||||||
|
|
||||||
|
print("Completed Python test case #1")
|
||||||
Loading…
Reference in New Issue