From 130f7ad4981f7ca22bb32ae9e58e4b72a431af48 Mon Sep 17 00:00:00 2001 From: meerkat Date: Fri, 5 Nov 2021 21:59:56 +1100 Subject: [PATCH] Refactor and improve Go processing --- docs/mkdocs.yml | 6 +- docs/source/samples/README.md | 27 ++- .../samples/{ => json}/BSBDirectoryFtp.json | 0 .../samples/{ => json}/BSBDirectoryHttp.json | 0 docs/source/samples/json/GEN005.ini | 55 +++++ docs/source/samples/json/README.md | 5 + .../samples/{ => json}/asic_ckan_api.json | 0 docs/source/samples/json/sample_01.md | 2 + docs/source/samples/json/sample_03.md | 2 + docs/source/samples/json/sample_04.md | 214 ++++++++++++++++++ .../powershell/Invoke-SampleGenerateBsb.ps1 | 8 +- .../Invoke-SampleGenerateBsbSecure.ps1 | 2 +- .../samples/python/SampleFetchFtpBsb.py | 2 +- .../samples/python/SampleFetchHttpBsb.py | 2 +- .../samples/python/SampleGenerateBsbSecure.py | 92 -------- ...GenerateBsb.py => SampleGenerateFtpBsb.py} | 35 +-- .../samples/python/SampleGenerateHttpBsb.py | 120 ++++++++++ docs/source/samples/python/listfiles_bsb.txt | 8 + source/golang/client/src/config/martilq.ini | 14 +- source/golang/client/src/main.go | 13 +- .../golang/client/src/martilq/attributes.go | 27 ++- source/golang/client/src/martilq/config.go | 72 ++++-- source/golang/client/src/martilq/marti.go | 30 ++- source/golang/client/src/martilq/resource.go | 66 +++++- source/powershell/MartiLQ.ps1 | 2 +- source/python/client/martiLQ.py | 4 +- test/powershell/test_MartiLQ.ps1 | 12 +- test/powershell/test_MartiLQCkan.ps1 | 6 +- test/python/test_MartiLQ1.py | 8 +- 29 files changed, 661 insertions(+), 173 deletions(-) rename docs/source/samples/{ => json}/BSBDirectoryFtp.json (100%) rename docs/source/samples/{ => json}/BSBDirectoryHttp.json (100%) create mode 100644 docs/source/samples/json/GEN005.ini rename docs/source/samples/{ => json}/asic_ckan_api.json (100%) create mode 100644 docs/source/samples/json/sample_04.md delete mode 100644 docs/source/samples/python/SampleGenerateBsbSecure.py rename docs/source/samples/python/{SampleGenerateBsb.py => SampleGenerateFtpBsb.py} (74%) create mode 100644 docs/source/samples/python/SampleGenerateHttpBsb.py create mode 100644 docs/source/samples/python/listfiles_bsb.txt diff --git a/docs/mkdocs.yml b/docs/mkdocs.yml index 4bdfd05..52275c5 100644 --- a/docs/mkdocs.yml +++ b/docs/mkdocs.yml @@ -1,6 +1,7 @@ -site_name: martiLQ +site_name: martiLQ Framework site_url: https://martilq.readthedocs.com/ + docs_dir: source nav: - Home: README.md @@ -13,7 +14,8 @@ nav: - Attributes: attributes.md - Custom: custom.md - Quality: quality.md - - Samples: samples/json/README.md + - Samples Code: samples/README.md + - Samples JSON: samples/json/README.md - Comparison: comparison.md - CKAN: ckan.md - MAGDA: magda.md diff --git a/docs/source/samples/README.md b/docs/source/samples/README.md index 515ba65..bbe3250 100644 --- a/docs/source/samples/README.md +++ b/docs/source/samples/README.md @@ -1,4 +1,29 @@ # Sample implementations -The following samples are provided +The following samples are provided. +## Python + +There are 3 sample Python programs: + + + * ``SampleGenerateFtpBsb.py`` that creates a **martiLQ** document from fetched BSB files FTP + * ``SampleGenerateHttpBsb.py`` that creates a **martiLQ** document from fetched BSB files over HTTP + * ``SampleFetchFtpBsb.py`` using FTP to fetch the data files and reconcile against canned + **martiLQ** document + * ``SampleFetchHttpBsb.py`` using HTTP to fetch the data files and reconcile against canned + **martiLQ** document + +The sample data files are Australian BSB files produced by the +Australian Payment Network. + + +## PowerShell + +There is a sample Python program named `` `` + + +## Pre Executed sample martiLQ document JSON file + +For samples of actual **martiLQ** documents +see [json/README.md](json/README.md) diff --git a/docs/source/samples/BSBDirectoryFtp.json b/docs/source/samples/json/BSBDirectoryFtp.json similarity index 100% rename from docs/source/samples/BSBDirectoryFtp.json rename to docs/source/samples/json/BSBDirectoryFtp.json diff --git a/docs/source/samples/BSBDirectoryHttp.json b/docs/source/samples/json/BSBDirectoryHttp.json similarity index 100% rename from docs/source/samples/BSBDirectoryHttp.json rename to docs/source/samples/json/BSBDirectoryHttp.json diff --git a/docs/source/samples/json/GEN005.ini b/docs/source/samples/json/GEN005.ini new file mode 100644 index 0000000..41e4578 --- /dev/null +++ b/docs/source/samples/json/GEN005.ini @@ -0,0 +1,55 @@ + +[General] + +logPath = +tempPath = +dataPath = + +[MartiLQ] + +tags = +publisher = +contactPoint = +accessLevel = +rights = +license = +batch = +theme = + + +[Resources] + +author = Meerkat +title = {{documentName}} +state = expired +expires = m:0:1:0 +encoding = +version = +urlPrefix = http://apnedata.merebox.com.s3.ap-southeast-2.amazonaws.com/au/bsb/ + +[Hash] + +hashAlgorithm = MD5 +signKey_File = +signKey_Password = + + +[Network] + +proxy = +username = +password = + + +[Custom_Spatial] + +enabled = false +country = +region = +town = + +[Custom_Temporal] + +enabled = false +businessDate = {{yesterday}} +runDate = {{today}} diff --git a/docs/source/samples/json/README.md b/docs/source/samples/json/README.md index d00ccec..d740703 100644 --- a/docs/source/samples/json/README.md +++ b/docs/source/samples/json/README.md @@ -3,9 +3,14 @@ This folder contains a number of sample **martiLQ** documents or configuration files. +The samples demonstrate the simplicity of generating the +**martiLQ** documents and gives insight as to construct +of the JSON document. + File|Content ---|--- [sample_01.md](sample_01.md)|Basic output [sample_02.md](sample_02.md)|Basic ouput with configuration file [sample_03.md](sample_03.md)|Directory output +[sample_04.md](sample_04.md)|Directory recursive generation with filter diff --git a/docs/source/samples/asic_ckan_api.json b/docs/source/samples/json/asic_ckan_api.json similarity index 100% rename from docs/source/samples/asic_ckan_api.json rename to docs/source/samples/json/asic_ckan_api.json diff --git a/docs/source/samples/json/sample_01.md b/docs/source/samples/json/sample_01.md index d16dd55..aa717a9 100644 --- a/docs/source/samples/json/sample_01.md +++ b/docs/source/samples/json/sample_01.md @@ -10,6 +10,8 @@ GOLANG program arguments are : -t GEN -m test_Sample01.json -s docs/source/martiLQ.md --title "GEN001" --description "Simple example with no config" ``` +The **martiLQ** document + ```json { "content-type": "application/vnd.martilq.json", diff --git a/docs/source/samples/json/sample_03.md b/docs/source/samples/json/sample_03.md index 3fdc3c1..fa32f6b 100644 --- a/docs/source/samples/json/sample_03.md +++ b/docs/source/samples/json/sample_03.md @@ -12,6 +12,8 @@ GOLANG program arguments are : -t GEN -m test_Sample03.json -c GEN002.ini -s docs/source/ --title "GEN003" --description "Directory example" ``` +The **martiLQ** document + ```json { "content-type": "application/vnd.martilq.json", diff --git a/docs/source/samples/json/sample_04.md b/docs/source/samples/json/sample_04.md new file mode 100644 index 0000000..fda3b31 --- /dev/null +++ b/docs/source/samples/json/sample_04.md @@ -0,0 +1,214 @@ +# Sample with entire directory + +The sample **martiLQ** document below has been generated +using the client GOLANG program and including all files +in a folder and child folders and matching the filter. + +The source is the project folder ``/docs/source`` and +default configuration has been used. + +This sample demonstrates the use of filters to select +what files to include. + +GOLANG program arguments are : +``` +-t GEN -m test_Sample04.json -s docs/source/ --title "GEN004" --description "Directory example with filter" -R --filter "^[r|R].*\.md" +``` + +The **martiLQ** document + +```json +{ + "content-type": "application/vnd.martilq.json", + "title": "GEN004", + "uid": "207096a1-aac1-4b66-9748-707f07b63691", + "description": "Directory example with filter", + "modified": "2021-11-03T22:02:42.8874753+11:00", + "publisher": "", + "contactPoint": "", + "accessLevel": "", + "rights": "", + "tags": null, + "license": "", + "state": "", + "batch": 0, + "describedBy": "", + "landingPage": "", + "theme": "", + "resources": [ + { + "title": "README.md", + "uid": "a3035230-d2b3-4157-87db-0e65df433686", + "documentName": "README.md", + "issueDate": "2021-11-03T22:02:42.8815605+11:00", + "modified": "2021-11-01T22:33:07.3888826+11:00", + "expires": "2028-11-03T00:00:00+11:00", + "state": "active", + "author": "", + "length": 1566, + "hash": { + "algo": "SHA256", + "value": "cf7cbf4186779984144fd022f4ba77b9773ffdc799b4a87f74e0ad2c52a7a261", + "signed": false + }, + "description": "", + "url": "file://README.md", + "version": "", + "content-type": "", + "encoding": "UTF-8", + "compression": "", + "encryption": "", + "describedBy": "", + "attributes": [ + { + "category": "dataset", + "name": "records", + "function": "count", + "comparison": "EQ", + "value": "43" + } + ] + }, + { + "title": "references.md", + "uid": "5e64cb84-bcb3-4002-9863-c2f5f8c4a29c", + "documentName": "references.md", + "issueDate": "2021-11-03T22:02:42.883191+11:00", + "modified": "2021-11-01T23:25:15.048493+11:00", + "expires": "2028-11-03T00:00:00+11:00", + "state": "active", + "author": "", + "length": 722, + "hash": { + "algo": "SHA256", + "value": "659db48c3b80a7c1ca3443fc43c45fe76eaa09c151e15f4dc1825fc84653f065", + "signed": false + }, + "description": "", + "url": "file://references.md", + "version": "", + "content-type": "", + "encoding": "UTF-8", + "compression": "", + "encryption": "", + "describedBy": "", + "attributes": [ + { + "category": "dataset", + "name": "records", + "function": "count", + "comparison": "EQ", + "value": "24" + } + ] + }, + { + "title": "resources.md", + "uid": "42632c8d-bd18-4c3f-9daa-5873fb547f8d", + "documentName": "resources.md", + "issueDate": "2021-11-03T22:02:42.883191+11:00", + "modified": "2021-11-01T23:23:55.517501+11:00", + "expires": "2028-11-03T00:00:00+11:00", + "state": "active", + "author": "", + "length": 4646, + "hash": { + "algo": "SHA256", + "value": "d2918d7b373a8b3bf43f377bc3d8df21f623b3c2a61bf5a16e19602fa763b8fc", + "signed": false + }, + "description": "", + "url": "file://resources.md", + "version": "", + "content-type": "", + "encoding": "UTF-8", + "compression": "", + "encryption": "", + "describedBy": "", + "attributes": [ + { + "category": "dataset", + "name": "records", + "function": "count", + "comparison": "EQ", + "value": "98" + } + ] + }, + { + "title": "README.md", + "uid": "66623c75-f9fe-4138-bc3c-979f176dfe19", + "documentName": "README.md", + "issueDate": "2021-11-03T22:02:42.8842642+11:00", + "modified": "2021-10-09T08:14:26.6966073+11:00", + "expires": "2028-11-03T00:00:00+11:00", + "state": "active", + "author": "", + "length": 67, + "hash": { + "algo": "SHA256", + "value": "85c4bc066e7a0a2dd9098fa1f58e05e741489d014064b2e58b6602115a12e5af", + "signed": false + }, + "description": "", + "url": "file://samples/README.md", + "version": "", + "content-type": "", + "encoding": "UTF-8", + "compression": "", + "encryption": "", + "describedBy": "", + "attributes": [ + { + "category": "dataset", + "name": "records", + "function": "count", + "comparison": "EQ", + "value": "4" + } + ] + }, + { + "title": "README.md", + "uid": "d661a0c3-fcf4-48f1-aee6-bdb28f996515", + "documentName": "README.md", + "issueDate": "2021-11-03T22:02:42.8847953+11:00", + "modified": "2021-11-03T00:04:54.642+11:00", + "expires": "2028-11-03T00:00:00+11:00", + "state": "active", + "author": "", + "length": 284, + "hash": { + "algo": "SHA256", + "value": "59717266e6cd617cece0c4104e620ec62dbc816042f58f0b212adb954a14a83b", + "signed": false + }, + "description": "", + "url": "file://samples/json/README.md", + "version": "", + "content-type": "", + "encoding": "UTF-8", + "compression": "", + "encryption": "", + "describedBy": "", + "attributes": [ + { + "category": "dataset", + "name": "records", + "function": "count", + "comparison": "EQ", + "value": "11" + } + ] + } + ], + "custom": [ + { + "extension": "software", + "softwareName": "MARTILQREFERENCE", + "author": "Meerkat@merebox.com", + "version": "0.0.1" + } + ] +} +``` diff --git a/docs/source/samples/powershell/Invoke-SampleGenerateBsb.ps1 b/docs/source/samples/powershell/Invoke-SampleGenerateBsb.ps1 index e3b63c0..7cc6e7a 100644 --- a/docs/source/samples/powershell/Invoke-SampleGenerateBsb.ps1 +++ b/docs/source/samples/powershell/Invoke-SampleGenerateBsb.ps1 @@ -113,7 +113,7 @@ ForEach ($item in $fileList) { } } -$fileJson = Join-Path -Path $localDirectory -ChildPath "MartiBSBRemote.mti" +$fileJson = Join-Path -Path $localDirectory -ChildPath "MartiBSBRemote.json" $oMarti | ConvertTo-Json -depth 100 | Out-File $fileJson Write-Host "Remote martiLQ definition file is $fileJson " -ForeGroundColor Green @@ -155,7 +155,7 @@ $oResource = New-MartiResource -SourcePath $zipFile -UrlPath $localDirectory -Lo Set-AttributeValueString -Attributes $oResource.attributes -Key "compression" -Category "format" -Function "algo" -Value "WINZIP" $oMarti.resources += $oResource -$fileJson = Join-Path -Path $localDirectory -ChildPath "MartiBSBZip.mti" +$fileJson = Join-Path -Path $localDirectory -ChildPath "MartiBSBZip.json" $oMarti | ConvertTo-Json -depth 100 | Out-File $fileJson Write-Host "ZIP martiLQ definition file is $fileJson " -ForeGroundColor Green @@ -170,7 +170,7 @@ $oMarti.contactPoint = "meerkat@merebox.com" $oMarti.landingPage = "https://github.com/meerkat-manor/marti/blob/draft_specifications/docs/samples/powershell/Invoke-BSBSample.ps1" $oMarti.theme = "payment" -$fileJson = Join-Path -Path $localDirectory -ChildPath "MartiBSBLocal.mti" +$fileJson = Join-Path -Path $localDirectory -ChildPath "MartiBSBLocal.json" $oMarti | ConvertTo-Json -depth 100 | Out-File $fileJson Write-Host "Local martiLQ definition file is $fileJson " -ForeGroundColor Green @@ -227,7 +227,7 @@ $oResource.encryption = New-Encryption -Algorithm "Passphrase" -Value $secret Set-AttributeValueString -Attributes $oResource.attributes -Key "compression" -Category "format" -Function "algo" -Value "7ZIP" $oMarti.resources += $oResource -$fileJson = Join-Path -Path $localDirectory -ChildPath "MartiBSBSecure.mti" +$fileJson = Join-Path -Path $localDirectory -ChildPath "MartiBSBSecure.json" $oMarti | ConvertTo-Json -depth 100 | Out-File $fileJson Write-Host "Secure 7ZIP martiLQ definition file is $fileJson " -ForeGroundColor Green diff --git a/docs/source/samples/powershell/Invoke-SampleGenerateBsbSecure.ps1 b/docs/source/samples/powershell/Invoke-SampleGenerateBsbSecure.ps1 index dcaf236..3b4bcd9 100644 --- a/docs/source/samples/powershell/Invoke-SampleGenerateBsbSecure.ps1 +++ b/docs/source/samples/powershell/Invoke-SampleGenerateBsbSecure.ps1 @@ -164,7 +164,7 @@ Write-Debug "Secret: $secret" Set-AttributeValueString -Attributes $oResource.attributes -Key "compression" -Category "format" -Function "algo" -Value "7ZIP" $oMarti.resources += $oResource -$fileJson = Join-Path -Path $localDirectory -ChildPath "MartiBSB7ZipPKI.mti" +$fileJson = Join-Path -Path $localDirectory -ChildPath "MartiBSB7ZipPKI.json" $oMarti | ConvertTo-Json -depth 50 | Out-File $fileJson Write-Host "7ZIP martiLQ definition file is $fileJson " -ForeGroundColor Green diff --git a/docs/source/samples/python/SampleFetchFtpBsb.py b/docs/source/samples/python/SampleFetchFtpBsb.py index cd3127e..df4700a 100644 --- a/docs/source/samples/python/SampleFetchFtpBsb.py +++ b/docs/source/samples/python/SampleFetchFtpBsb.py @@ -3,7 +3,7 @@ import os import sys import csv -sys.path.insert(0, "../../../source/python/client") +sys.path.insert(0, "../../../../source/python/client") from martiLQ import * test_directory = "./test/fetch_ftp" diff --git a/docs/source/samples/python/SampleFetchHttpBsb.py b/docs/source/samples/python/SampleFetchHttpBsb.py index c60bf9b..1de5149 100644 --- a/docs/source/samples/python/SampleFetchHttpBsb.py +++ b/docs/source/samples/python/SampleFetchHttpBsb.py @@ -3,7 +3,7 @@ import os import sys import csv -sys.path.insert(0, "../../../source/python/client") +sys.path.insert(0, "../../../../source/python/client") from martiLQ import * test_directory = "./test/fetch_http" diff --git a/docs/source/samples/python/SampleGenerateBsbSecure.py b/docs/source/samples/python/SampleGenerateBsbSecure.py deleted file mode 100644 index aa48be2..0000000 --- a/docs/source/samples/python/SampleGenerateBsbSecure.py +++ /dev/null @@ -1,92 +0,0 @@ - -import ftplib -import os -import json -import sys -import csv - -sys.path.insert(0, "../../../source/python/client") -from martiLQ import * - -def ftpList(host, path): - - files = [] - with ftplib.FTP(host) as ftp: - try: - ftp.login() - ftp.cwd(path) - files = ftp.nlst() - except ftplib.all_errors as e: - print('FTP error:', e) - - return files - - -def ftpPull(host, file_remote, file_local): - - with ftplib.FTP(host) as ftp: - - try: - ftp.login() - - with open(file_local, 'wb') as fl: - res = ftp.retrbinary(f"RETR {file_remote}", fl.write) - if not res.startswith('226 Transfer complete'): - print('Download failed') - if os.path.isfile(file_local): - os.remove(file_local) - - except ftplib.all_errors as e: - print('FTP error:', e) - if os.path.isfile(file_local): - os.remove(file_local) - - -remote_host = "bsb.hostedftp.com" -remote_dir = "/~auspaynetftp/BSB/" - -print("Fetch sample file list") -files = ftpList(remote_host, remote_dir) - -if not os.path.exists("./test"): - os.mkdir("./test") - -print("Fetch sample files") -for file_name in files: - if file_name.startswith("BSBDirectory"): - if file_name.endswith(".csv") | file_name.endswith(".txt"): - file_remote = remote_dir + file_name - file_local = "./test/" + file_name - ftpPull(remote_host, file_remote, file_local) - -print("Creating martiLQ definition") -mlq = martiLQ() -oMarti = mlq.NewMartiDefinition() - -for file_name in files: - if file_name.startswith("BSBDirectory"): - if file_name.endswith(".csv") | file_name.endswith(".txt"): - oResource = mlq.NewMartiLQResource(os.path.join("./test/", file_name), "", False, True, "./test/logs") - oMarti["resources"].append(oResource) - -mlq.CloseLog() -print("Save martiLQ definition") -jd = json.dumps(oMarti, indent=5) - -jsonFile = open("./test/BSBDirectoryPlain.mti", "w") -jsonFile.write(jd) -jsonFile.close() -print("Sample completed: SampleGenerateBsb.py") - -lqresults, testError = mlq.TestMartiDefinition("./test/BSBDirectoryPlain.mti") - -testfile = open("./test/LoadQualityTest01.csv", "w+", newline ="") -with testfile: - lqwriter = csv.writer(testfile) - lqwriter.writerows(lqresults) - -if testError: - print("MISMATCH DETECTED") - -print("Test completed: SampleGenerateBsb.py") - diff --git a/docs/source/samples/python/SampleGenerateBsb.py b/docs/source/samples/python/SampleGenerateFtpBsb.py similarity index 74% rename from docs/source/samples/python/SampleGenerateBsb.py rename to docs/source/samples/python/SampleGenerateFtpBsb.py index e4c7671..4cb0b35 100644 --- a/docs/source/samples/python/SampleGenerateBsb.py +++ b/docs/source/samples/python/SampleGenerateFtpBsb.py @@ -7,7 +7,7 @@ import csv import zipfile -sys.path.insert(0, "../../../source/python/client") +sys.path.insert(0, "../../../../source/python/client") from martiLQ import * ftpFetch = True @@ -53,15 +53,16 @@ remote_dir = "/~auspaynetftp/BSB/" print("Fetch sample file list") files = ftpList(remote_host, remote_dir) -if not os.path.exists("./test"): - os.mkdir("./test") +test_dir = "./test/ftp" +if not os.path.exists(test_dir): + os.mkdir(test_dir) print("Fetch sample files") for file_name in files: if file_name.startswith("BSBDirectory"): if file_name.endswith(".csv") | file_name.endswith(".txt"): file_remote = remote_dir + file_name - file_local = "./test/" + file_name + file_local = os.path.join(test_dir, file_name) if ftpFetch: ftpPull(remote_host, file_remote, file_local) @@ -72,17 +73,17 @@ oMarti = mlq.NewMartiDefinition() for file_name in files: if file_name.startswith("BSBDirectory"): if file_name.endswith(".csv") | file_name.endswith(".txt"): - oResource = mlq.NewMartiLQResource(os.path.join("./test/", file_name), "", False, True) + oResource = mlq.NewMartiLQResource(os.path.join(test_dir, file_name), "", False, True) oMarti["resources"].append(oResource) mlq.CloseLog() print("Save martiLQ definition") -jsonFile = open("./test/BSBDirectoryPlain.mti", "w") +jsonFile = open(os.path.join(test_dir, "BSBDirectoryPlain.json"), "w") jsonFile.write(json.dumps(oMarti, indent=5)) jsonFile.close() -print("Base sample mti written: BSBDirectoryPlain.mti") +print("Base sample JSON written: BSBDirectoryPlain.json") print("Creating martiLQ ZIP file") @@ -91,23 +92,23 @@ fileZipCount = 0 mlq = martiLQ() oMarti = mlq.NewMartiDefinition() -with zipfile.ZipFile("./test/" + zipFileName, "w", compression=zipfile.ZIP_DEFLATED) as zipObj: +with zipfile.ZipFile(os.path.join(test_dir, zipFileName), "w", compression=zipfile.ZIP_DEFLATED) as zipObj: for file_name in files: if file_name.startswith("BSBDirectory"): if file_name.endswith(".csv") | file_name.endswith(".txt"): file_remote = remote_dir + file_name - file_local = "./test/" + file_name + file_local = os.path.join(test_dir, file_name) if ftpFetch: ftpPull(remote_host, file_remote, file_local) zipObj.write(file_local, file_name) fileZipCount = fileZipCount + 1 - oResource = mlq.NewMartiLQResource(os.path.join("./test/", file_name), "", False, True) + oResource = mlq.NewMartiLQResource(os.path.join(test_dir, file_name), "", False, True) oResource["url"] = "@"+zipFileName + "/" + file_name oMarti["resources"].append(oResource) -oResource = mlq.NewMartiLQResource(os.path.join("./test/", zipFileName), "", False, True) -oResource["url"] = "./test/" + zipFileName +oResource = mlq.NewMartiLQResource(os.path.join(test_dir, zipFileName), "", False, True) +oResource["url"] = os.path.join(test_dir, zipFileName) mlq.SetAttributeValueString(Attributes=oResource["attributes"], Key="compression", Category="format", Function="algo", Value="WINZIP") mlq.SetAttributeValueNumber(Attributes=oResource["attributes"], Key="files", Category="dataset", Function="count", Value=fileZipCount) oMarti["resources"].append(oResource) @@ -115,18 +116,18 @@ oMarti["resources"].append(oResource) mlq.CloseLog() print("Save martiLQ ZIP definition") -jsonFile = open("./test/MartiLQ_BSBZip.mti", "w") +jsonFile = open(os.path.join(test_dir, "MartiLQ_BSBZip.json"), "w") jsonFile.write(json.dumps(oMarti, indent=5)) jsonFile.close() -print("ZIP sample mti written: MartiLQ_BSBZip.mti") +print("ZIP sample JSON written: MartiLQ_BSBZip.json") print("Sample completed: SampleGenerateBsb.py") -lqresults, testError = mlq.TestMartiDefinition("./test/BSBDirectoryPlain.mti") +lqresults, testError = mlq.TestMartiDefinition(os.path.join(test_dir, "BSBDirectoryPlain.json")) -testfile = open("./test/LoadQualityTest01.csv", "w+", newline ="") +testfile = open(os.path.join(test_dir, "LoadQualityTest01.csv"), "w+", newline ="") with testfile: lqwriter = csv.writer(testfile) lqwriter.writerows(lqresults) @@ -134,5 +135,5 @@ with testfile: if testError: print("MISMATCH DETECTED") -print("Test completed: SampleGenerateBsb.py") +print("Test completed: SampleGenerateFtpBsb.py") diff --git a/docs/source/samples/python/SampleGenerateHttpBsb.py b/docs/source/samples/python/SampleGenerateHttpBsb.py new file mode 100644 index 0000000..ecee4fb --- /dev/null +++ b/docs/source/samples/python/SampleGenerateHttpBsb.py @@ -0,0 +1,120 @@ + +import os +import sys +import urllib.request +import shutil +import json +import csv +import zipfile +import datetime +import time + +sys.path.insert(0, "../../../../source/python/client") +from martiLQ import * + +httpFetch = True +os.environ["MARTILQ_LOGPATH"] = "./test/logs" + + +def HttpList(remote_url): + + files = [] + + with open("listfiles_bsb.txt", "r") as f: + files = f.read().splitlines() + + return files + + + +remote_url = "http://apnedata.merebox.com.s3.ap-southeast-2.amazonaws.com/au/bsb/" +print("Fetch sample file list") +files = HttpList(remote_url) + +test_dir = "./test/http" +if not os.path.exists(test_dir): + os.mkdir(test_dir) + +if httpFetch: + print("Fetch sample files") + for file_name in files: + if file_name.startswith("BSBDirectory"): + if file_name.endswith(".csv") | file_name.endswith(".txt"): + with urllib.request.urlopen(remote_url + file_name) as resp: + last_modified = resp.info()["Last-Modified"] + dt_obj = datetime.datetime.strptime(last_modified, '%a, %d %b %Y %H:%M:%S %Z') + + data_file_name = os.path.join(test_dir, file_name) + with open(data_file_name, 'wb') as data_file: + shutil.copyfileobj(resp, data_file) + + modTime = time.mktime(dt_obj.timetuple()) + os.utime(data_file_name, (modTime, modTime)) + +print("Creating martiLQ definition") +mlq = martiLQ() +oMarti = mlq.NewMartiDefinition() + +for file_name in files: + if file_name.startswith("BSBDirectory"): + if file_name.endswith(".csv") | file_name.endswith(".txt"): + oResource = mlq.NewMartiLQResource(os.path.join(test_dir, file_name), "", False, True) + oMarti["resources"].append(oResource) + +mlq.CloseLog() + + +print("Save martiLQ definition") +jsonFile = open(os.path.join(test_dir, "BSBDirectoryPlain.json"), "w") +jsonFile.write(json.dumps(oMarti, indent=5)) +jsonFile.close() +print("Base sample JSON written: BSBDirectoryPlain.json") + +print("Creating martiLQ ZIP file") +zipFileName = "BSBDirectory.zip" +fileZipCount = 0 + +mlq = martiLQ() +oMarti = mlq.NewMartiDefinition() +with zipfile.ZipFile(os.path.join(test_dir, zipFileName), "w", compression=zipfile.ZIP_DEFLATED) as zipObj: + for file_name in files: + if file_name.startswith("BSBDirectory"): + if file_name.endswith(".csv") | file_name.endswith(".txt"): + file_local = os.path.join(test_dir, file_name) + zipObj.write(file_local, file_name) + fileZipCount = fileZipCount + 1 + oResource = mlq.NewMartiLQResource(os.path.join(test_dir, file_name), "", False, True) + oResource["url"] = "@"+zipFileName + "/" + file_name + oMarti["resources"].append(oResource) + + +oResource = mlq.NewMartiLQResource(os.path.join(test_dir, zipFileName), "", False, True) +oResource["url"] = test_dir + zipFileName +mlq.SetAttributeValueString(Attributes=oResource["attributes"], Key="compression", Category="format", Function="algo", Value="WINZIP") +mlq.SetAttributeValueNumber(Attributes=oResource["attributes"], Key="files", Category="dataset", Function="count", Value=fileZipCount) +oMarti["resources"].append(oResource) + +mlq.CloseLog() + +print("Save martiLQ ZIP definition") +jsonFile = open(os.path.join(test_dir, "MartiLQ_BSBZip.json"), "w") +jsonFile.write(json.dumps(oMarti, indent=5)) +jsonFile.close() +print("ZIP sample JSON written: MartiLQ_BSBZip.json") + + + +print("Sample completed: SampleGenerateBsb.py") + +lqresults, testError = mlq.TestMartiDefinition(os.path.join(test_dir, "BSBDirectoryPlain.json")) + +testfile = open(os.path.join(test_dir, "LoadQualityTest01.csv"), "w+", newline ="") +with testfile: + lqwriter = csv.writer(testfile) + lqwriter.writerows(lqresults) + +if testError: + print("MISMATCH DETECTED") + +print("Test completed: SampleGenerateFtpBsb.py") + diff --git a/docs/source/samples/python/listfiles_bsb.txt b/docs/source/samples/python/listfiles_bsb.txt new file mode 100644 index 0000000..801e35b --- /dev/null +++ b/docs/source/samples/python/listfiles_bsb.txt @@ -0,0 +1,8 @@ +BSBDirectoryOct21-307.csv +BSBDirectorySep21-306.csv +BSBDirectoryAug21-305.csv +BSBDirectoryJul21-304.csv +BSBDirectoryJun21-303.csv +BSBDirectoryMay21-302.csv +BSBDirectoryApr21-301.csv +BSBDirectoryMar21-300.csv \ No newline at end of file diff --git a/source/golang/client/src/config/martilq.ini b/source/golang/client/src/config/martilq.ini index b9e6fd8..83ebbae 100644 --- a/source/golang/client/src/config/martilq.ini +++ b/source/golang/client/src/config/martilq.ini @@ -13,19 +13,19 @@ contactPoint = accessLevel = rights = license = -batch = @./config/batch.no +batch = theme = [Resources] author = -title = {{documentName}} +title = state = -expires = 2:0:0 -encoding = UTF-8 +expires = +encoding = version = -urlPrefix = http://localhost/martilq/ +urlPrefix = [Hash] @@ -44,9 +44,9 @@ password = [Custom_Spatial] enabled = false -country = Netherland +country = region = -town = Amsterdam +town = [Custom_Temporal] diff --git a/source/golang/client/src/main.go b/source/golang/client/src/main.go index 13a7d8b..210c226 100644 --- a/source/golang/client/src/main.go +++ b/source/golang/client/src/main.go @@ -17,6 +17,7 @@ type Parameters struct { task string sourcePath string recursive bool + filter string update bool urlPrefix string configPath string @@ -121,6 +122,16 @@ func loadArguments(args []string) { panic("Missing parameter for TITLE") } } + + if args[ix] == "--filter" { + matched = true + if ix < maxArgs { + ix = ix + 1 + params.filter = args[ix] + } else { + panic("Missing parameter for FILTER") + } + } if args[ix] == "--description" { matched = true @@ -244,7 +255,7 @@ func main () { panic("MartiLQ document '"+ params.definitionPath+"' already exists and update not specified") } - m := martilq.ProcessFilePath(params.configPath, params.sourcePath, params.recursive, params.urlPrefix, params.definitionPath ) + m := martilq.ProcessFilePath(params.configPath, params.sourcePath, params.filter, params.recursive, params.urlPrefix, params.definitionPath ) if params.title != "" { m.Title = params.title } diff --git a/source/golang/client/src/martilq/attributes.go b/source/golang/client/src/martilq/attributes.go index 876606b..74ad027 100644 --- a/source/golang/client/src/martilq/attributes.go +++ b/source/golang/client/src/martilq/attributes.go @@ -6,6 +6,7 @@ import ( "io" "bytes" "strconv" + "log" ) // These are predefined categories, custom ones can be added @@ -160,19 +161,37 @@ func NewDefaultTemporalAttributes(businessDate time.Time, runDate time.Time, dur } -func SetMartiAttribute(Attributes []Attribute, ACategory string, AName string, AFunction string, Comparison string , Value string) { +func RemoveMartiAttribute(Attributes []Attribute, ACategory string, AName string, AFunction string, Comparison string , Value string) []Attribute { + + for ix := 0; ix< len(Attributes); ix++ { + attr := Attributes[ix] + if attr.Category == ACategory && attr.Name == AName && attr.Function == AFunction && attr.Comparison == Comparison { + copy(Attributes[ix:], Attributes[ix+1:]) + Attributes[len(Attributes)-1] = Attribute{} + return Attributes[:len(Attributes)-1] + } + } + + log.Fatal("No matching record found") + return Attributes +} + +func SetMartiAttribute(Attributes []Attribute, ACategory string, AName string, AFunction string, Comparison string , Value string) []Attribute { for ix := 0; ix< len(Attributes); ix++ { attr := Attributes[ix] if attr.Category == ACategory && attr.Name == AName && attr.Function == AFunction { - Attributes[ix].Comparison = Comparison - Attributes[ix].Value = Value - return + if attr.Comparison == Comparison || attr.Comparison == "NA" { + Attributes[ix].Comparison = Comparison + Attributes[ix].Value = Value + return Attributes + } } } oAttribute := Attribute { ACategory, AName, AFunction, Comparison, Value } Attributes = append(Attributes, oAttribute) + return Attributes } diff --git a/source/golang/client/src/martilq/config.go b/source/golang/client/src/martilq/config.go index 8419133..f9bd7ff 100644 --- a/source/golang/client/src/martilq/config.go +++ b/source/golang/client/src/martilq/config.go @@ -3,6 +3,7 @@ package martilq import ( "fmt" "gopkg.in/ini.v1" + "log" "os" "time" "strings" @@ -13,8 +14,8 @@ const cSoftwareName = "MARTILQREFERENCE" const cSoftwareAuthor = "Meerkat@merebox.com" const cSoftwareVersion = "0.0.1" const cIniFileName = "martilq.ini" -const cExpires = "7:0:0" -const cEncoding = "UTF-8" +const cExpires = "t:7:0:0" +const cEncoding = "" type configuration struct { softwareName string @@ -270,36 +271,79 @@ func Loadenv(key string, default_value string ) string { } -func (c *configuration) ExpireDate( ) time.Time { +func (c *configuration) ExpireDate(sourcePath string ) time.Time { var expires time.Time lExpires := strings.Split(c.expires,":") - if len(lExpires) != 3 && len(lExpires) != 6 { + if len(lExpires) != 4 && len(lExpires) != 7 { panic("Expires value '"+ c.expires +"' is invalid") } + base := lExpires[0] + if sourcePath == "" && base == "m" { + base = "t" + } + + modified := time.Now() + if base == "m" { + stats, err := os.Stat(sourcePath) + if err != nil { + if os.IsNotExist(err) { + log.Printf("'" + sourcePath + "' file does not exist for Expiry.") + base = "t" + } + } else { + modified = stats.ModTime() + } + } var lExpire [3]int - lex, _ := strconv.Atoi(lExpires[0]) + lex, _ := strconv.Atoi(lExpires[1]) lExpire[0] = lex - lex, _ =strconv.Atoi(lExpires[1]) - lExpire[1] = lex lex, _ =strconv.Atoi(lExpires[2]) + lExpire[1] = lex + lex, _ =strconv.Atoi(lExpires[3]) lExpire[2] = lex - if len(lExpires) > 3 { + if len(lExpires) > 4 { var lExpireD [3]int - lex, _ := strconv.Atoi(lExpires[3]) + lex, _ := strconv.Atoi(lExpires[4]) lExpireD[0] = lex - lex, _ =strconv.Atoi(lExpires[4]) - lExpireD[1] = lex lex, _ =strconv.Atoi(lExpires[5]) + lExpireD[1] = lex + lex, _ =strconv.Atoi(lExpires[6]) lExpireD[2] = lex - expires = time.Now().AddDate(lExpire[0],lExpire[1],lExpire[2]).Add(time.Hour * time.Duration(lExpireD[0])).Add(time.Minute * time.Duration(lExpireD[1])).Add(time.Second * time.Duration(lExpireD[2])) + + switch base { + case "m": + expires = modified.AddDate(lExpire[0],lExpire[1],lExpire[2]).Add(time.Hour * time.Duration(lExpireD[0])).Add(time.Minute * time.Duration(lExpireD[1])).Add(time.Second * time.Duration(lExpireD[2])) + case "r": + expires = c.temporal.RunDate.AddDate(lExpire[0],lExpire[1],lExpire[2]).Add(time.Hour * time.Duration(lExpireD[0])).Add(time.Minute * time.Duration(lExpireD[1])).Add(time.Second * time.Duration(lExpireD[2])) + case "b": + expires = c.temporal.BusinessDate.AddDate(lExpire[0],lExpire[1],lExpire[2]).Add(time.Hour * time.Duration(lExpireD[0])).Add(time.Minute * time.Duration(lExpireD[1])).Add(time.Second * time.Duration(lExpireD[2])) + case "t": + fallthrough + default: + expires = time.Now().AddDate(lExpire[0],lExpire[1],lExpire[2]).Add(time.Hour * time.Duration(lExpireD[0])).Add(time.Minute * time.Duration(lExpireD[1])).Add(time.Second * time.Duration(lExpireD[2])) + } } else { - expires = time.Now().AddDate(lExpire[0],lExpire[1],lExpire[2]) - expires = time.Date(expires.Year(), expires.Month(), expires.Day(), 0, 0, 0, 0, time.Local) + switch base { + case "m": + expires = modified.AddDate(lExpire[0],lExpire[1],lExpire[2]) + expires = time.Date(expires.Year(), expires.Month(), expires.Day(), 0, 0, 0, 0, time.Local) + case "r": + expires = c.temporal.RunDate.AddDate(lExpire[0],lExpire[1],lExpire[2]) + expires = time.Date(expires.Year(), expires.Month(), expires.Day(), 0, 0, 0, 0, time.Local) + case "b": + expires = c.temporal.BusinessDate.AddDate(lExpire[0],lExpire[1],lExpire[2]) + expires = time.Date(expires.Year(), expires.Month(), expires.Day(), 0, 0, 0, 0, time.Local) + case "t": + fallthrough + default: + expires = time.Now().AddDate(lExpire[0],lExpire[1],lExpire[2]) + expires = time.Date(expires.Year(), expires.Month(), expires.Day(), 0, 0, 0, 0, time.Local) + } } return expires diff --git a/source/golang/client/src/martilq/marti.go b/source/golang/client/src/martilq/marti.go index bcd810a..e5e59b7 100644 --- a/source/golang/client/src/martilq/marti.go +++ b/source/golang/client/src/martilq/marti.go @@ -16,6 +16,7 @@ import ( "bufio" "log" "reflect" + "regexp" ) @@ -140,13 +141,12 @@ func (m *Marti) LoadConfig(ConfigPath string) { } -func (m *Marti) AddResource(Title string, SourcePath string, Url string) (Resource, error) { +func (m *Marti) AddResource(SourcePath string, Url string) (Resource, error) { r, err := NewMartiLQResource(m.config, SourcePath, Url, false, true) if err != nil { return r, errors.New("Error in adding resource: "+SourcePath) } - r.Title = Title // Find if we already have the resource // This can occur if we are reloading @@ -184,7 +184,7 @@ func (m *Marti) Save(pathFile string) bool { } -func ProcessFilePath(ConfigPath string, SourcePath string, Recursive bool, UrlPrefix string, DefinitionPath string) Marti { +func ProcessFilePath(ConfigPath string, SourcePath string, Filter string, Recursive bool, UrlPrefix string, DefinitionPath string) Marti { m := NewMarti() @@ -228,22 +228,36 @@ func ProcessFilePath(ConfigPath string, SourcePath string, Recursive bool, UrlPr filepath.Walk(fileAbs, func(path string, info os.FileInfo, err error) error { if err != nil { log.Fatalf(err.Error()) + return nil } if info.IsDir() == false { diff := strings.Replace(path, diffCheck, "", -1) if Recursive || diff == info.Name() { - url := UrlPrefix+strings.Replace(diff, "\\", "/", -1) - if UrlPrefix[0:6] == "file://" || UrlPrefix[0:1] == "\\\\" { - url = UrlPrefix+diff + + found := true + if Filter != "" { + found, err = regexp.MatchString(Filter, info.Name()) + if err != nil { + log.Fatal(err) + } } - m.AddResource(info.Name(), path, url) + + if found { + url := UrlPrefix+strings.Replace(diff, "\\", "/", -1) + if UrlPrefix[0:6] == "file://" || UrlPrefix[0:1] == "\\\\" { + url = UrlPrefix+diff + } + m.AddResource(path, url) + } + } else { + return filepath.SkipDir } } return nil }) } else { url := UrlPrefix+fileStat.Name() - m.AddResource(fileStat.Name(), fileAbs, url) + m.AddResource(fileAbs, url) } } diff --git a/source/golang/client/src/martilq/resource.go b/source/golang/client/src/martilq/resource.go index ac7274d..5900c96 100644 --- a/source/golang/client/src/martilq/resource.go +++ b/source/golang/client/src/martilq/resource.go @@ -2,10 +2,16 @@ package martilq import ( "github.com/google/uuid" + "encoding/csv" "os" + "io" + "strings" "time" + "fmt" "log" "errors" + "mime" + "strconv" ) @@ -42,7 +48,7 @@ func NewResource(config configuration) Resource { r.IssueDate = time.Now() r.State = config.state r.Author = config.author - r.Expires = config.ExpireDate() + r.Expires = config.ExpireDate("") r.Encoding = config.encoding return r @@ -70,13 +76,26 @@ func NewMartiLQResource(config configuration, sourcePath string, urlPath string, r.State = config.state r.Author = config.author - r.Expires = config.ExpireDate() + r.Expires = config.ExpireDate(sourcePath) + if time.Now().Before(r.Expires) && r.State == "expired" { + r.State = "active" + } r.Encoding = config.encoding r.DocumentName = stats.Name() - if config.title == "{{documentName}}" { + switch config.title { + case "{{documentName.ext}}": r.Title = r.DocumentName + case "{{documentName}}": + parts := strings.Split(r.DocumentName, ".") + r.Title = strings.Replace(r.DocumentName, ("."+parts[len(parts)-1]), "",-1) + case "{{print}}": + fmt.Println("r: "+ r.Title) + r.Title = r.DocumentName + default: + r.Title = config.title } + r.IssueDate = time.Now() r.Modified = stats.ModTime() r.Url = urlPath @@ -86,7 +105,46 @@ func NewMartiLQResource(config configuration, sourcePath string, urlPath string, r.Hash = h } - r.Attributes = NewDefaultExtensionAttributes(sourcePath, extendAttributes) + parts := strings.Split(sourcePath,".") + extension := parts[len(parts)-1] + + r.Content_Type = mime.TypeByExtension("."+extension) + records := 0 + columns := -1 + + switch extension { + case "csv": + r.Attributes = NewDefaultCsvAttributes(true, ",") + + f_csv, err := os.Open(sourcePath) + if err != nil { + log.Fatal(err) + } + + rdr := csv.NewReader(f_csv) + for { + record, err := rdr.Read() + if err == io.EOF { + break + } + if err != nil { + log.Fatal(err) + } + records = records + 1 + if len(record) > columns { + columns = len(record) + } + } + f_csv.Close() + + default: + r.Attributes = NewDefaultExtensionAttributes(sourcePath, extendAttributes) + } + + if columns > 0 { + r.Attributes = SetMartiAttribute(r.Attributes, "dataset", "columns", "count", "EQ" , strconv.Itoa(columns)) + r.Attributes = SetMartiAttribute(r.Attributes, "dataset", "records", "count", "EQ" , strconv.Itoa(records)) + } return r, nil } diff --git a/source/powershell/MartiLQ.ps1 b/source/powershell/MartiLQ.ps1 index 0b437d1..da3f326 100644 --- a/source/powershell/MartiLQ.ps1 +++ b/source/powershell/MartiLQ.ps1 @@ -2,7 +2,7 @@ $script:LogPathName = "" $script:SoftwareVersion = "0.0.1" -$global:default_metaFile = "##marti##.mti" +$global:default_metaFile = "##marti##.json" function Get-LogName { diff --git a/source/python/client/martiLQ.py b/source/python/client/martiLQ.py index b3e8fd2..6bffe9b 100644 --- a/source/python/client/martiLQ.py +++ b/source/python/client/martiLQ.py @@ -16,7 +16,7 @@ import mimetypes class martiLQ: _SoftwareVersion = "0.0.1" - _default_metaFile = "##marti##.mti" + _default_metaFile = "##marti##.json" _oSoftware = { "extension": "software", @@ -870,7 +870,7 @@ class martiLQ: otest = ["ResourceName", "Level", "Metric", "Matches", "LocalCalculation", "SuppliedValue" ] oTestResults.append(otest) - otest = ["", "Batch", "Resource count", (len(self._oMartiDefinition["resources"]) == len(lq["resources"])), len(self._oMartiDefinition["resources"]), len(lq["resources"]) ] + otest = ["@", "Batch", "Resource count", (len(self._oMartiDefinition["resources"]) == len(lq["resources"])), len(self._oMartiDefinition["resources"]), len(lq["resources"]) ] oTestResults.append(otest) for resource in self._oMartiDefinition["resources"]: diff --git a/test/powershell/test_MartiLQ.ps1 b/test/powershell/test_MartiLQ.ps1 index 91dcfe0..053049a 100644 --- a/test/powershell/test_MartiLQ.ps1 +++ b/test/powershell/test_MartiLQ.ps1 @@ -9,7 +9,7 @@ try { $oMarti.description = "Sample execution" $x = ConvertTo-Json -InputObject $oMarti - Set-Content -Path ".\test\powershell\results\marti_test01.mti" -Value $x + Set-Content -Path ".\test\powershell\results\marti_test01.json" -Value $x Write-Host "Test case #2" $ArchiveFile = ".\test\powershell\results\marti_test02.zip" @@ -25,23 +25,23 @@ try { $oMarti.description = "Sample execution for ckan" $x = ConvertTo-Json -InputObject $oMarti - Set-Content -Path ".\test\powershell\results\marti_test02json.mti" -Value $x + Set-Content -Path ".\test\powershell\results\marti_test02json.json" -Value $x $x = ConvertTo-Csv -InputObject $oMarti - Set-Content -Path ".\test\powershell\results\marti_test02csv.mti." -Value $x + Set-Content -Path ".\test\powershell\results\marti_test02csv.json" -Value $x $x = ConvertTo-Xml -As String -InputObject $oMarti -Depth 6 - Set-Content -Path ".\test\powershell\results\marti_test02xml.mti" -Value $x + Set-Content -Path ".\test\powershell\results\marti_test02xml.json" -Value $x $x = ConvertTo-Html -InputObject $oMarti - Set-Content -Path ".\test\powershell\results\marti_test02html.mti" -Value $x + Set-Content -Path ".\test\powershell\results\marti_test02html.json" -Value $x Write-Host "Test case #5" $oMarti = New-MartiResource -SourcePath ".\docs\eror" -LogPath ".\test\powershell\results\Logs" $oMarti.description = "Sample execution with error" $x = ConvertTo-Json -InputObject $oMarti - Set-Content -Path ".\test\powershell\results\marti_test03.mti" -Value $x + Set-Content -Path ".\test\powershell\results\marti_test03.json" -Value $x } catch { diff --git a/test/powershell/test_MartiLQCkan.ps1 b/test/powershell/test_MartiLQCkan.ps1 index cdaeee6..9f01679 100644 --- a/test/powershell/test_MartiLQCkan.ps1 +++ b/test/powershell/test_MartiLQCkan.ps1 @@ -7,7 +7,7 @@ $ckan = Get-Content -Path ".\docs\samples\asic_ckan_api.json" -Raw $oMarti = ConvertFrom-Ckan -InputObject $ckan $x = ConvertTo-Json -InputObject $oMarti -Set-Content -Path ".\test\powershell\results\marti_test05.mti" -Value $x +Set-Content -Path ".\test\powershell\results\marti_test05.json" -Value $x $covid_1 = Invoke-WebRequest "https://data.nsw.gov.au/data/api/3/action/package_show?id=793ac07d-a5f4-4851-835c-3f7158c19d15" @@ -18,7 +18,7 @@ $oMarti.tags += "gov" $oMarti.tags += "nsw" $oMarti.publisher = "NSW government (Australia)" $x = ConvertTo-Json -InputObject $oMarti -Set-Content -Path ".\test\powershell\results\marti_test06.mti" -Value $x +Set-Content -Path ".\test\powershell\results\marti_test06.json" -Value $x # cases @@ -31,5 +31,5 @@ $oMarti.tags += "gov" $oMarti.tags += "nsw" $oMarti.publisher = "NSW government (Australia)" $x = ConvertTo-Json -InputObject $oMarti -Set-Content -Path ".\test\powershell\results\marti_test07.mti" -Value $x +Set-Content -Path ".\test\powershell\results\marti_test07.json" -Value $x diff --git a/test/python/test_MartiLQ1.py b/test/python/test_MartiLQ1.py index 6e6665a..33a1407 100644 --- a/test/python/test_MartiLQ1.py +++ b/test/python/test_MartiLQ1.py @@ -20,17 +20,17 @@ mlq.NewMartiChildItem(SourceFolder= "./docs/*", UrlPath="./docs" , ExcludeHash=F oMarti["description"] = "Sample execution #1" print("Save martiLQ definition #1") -mlq.Save("./test/python/results/DocsPlain1.mti") +mlq.Save("./test/python/results/DocsPlain1.json") print("Save martiLQ definition #2") oMarti["description"] = "Sample execution #2" -jsonFile = open("./test/python/results/DocsPlain2.mti", "w") +jsonFile = open("./test/python/results/DocsPlain2.json", "w") jsonFile.write(json.dumps(oMarti, indent=5)) jsonFile.close() -print("Base sample mti written: DocsPlain2.mti") +print("Base sample JSON written: DocsPlain2.json") print("Load martiLQ definition #1") -mlq.Load("./test/python/results/DocsPlain1.mti") +mlq.Load("./test/python/results/DocsPlain1.json") oMarti = mlq.Get() print("Definition description is: {}".format(oMarti["description"]))