Reworked and refactored code
Samples have been improved as has the Powershell configurationdraft_specifications
parent
28168c5fc1
commit
74d5474b32
|
|
@ -1,8 +1,8 @@
|
|||
|
||||
. ..\..\..\source\powershell\MartiLQ.ps1
|
||||
. ..\..\..\source\powershell\MartiLQConfiguration.ps1
|
||||
. ..\..\..\source\powershell\MartiLQResource.ps1
|
||||
. ..\..\..\source\powershell\MartiLQAttribute.ps1
|
||||
. .\source\powershell\MartiLQ.ps1
|
||||
. .\source\powershell\MartiLQConfiguration.ps1
|
||||
. .\source\powershell\MartiLQResource.ps1
|
||||
. .\source\powershell\MartiLQAttribute.ps1
|
||||
|
||||
|
||||
function PullFtpFile {
|
||||
|
|
@ -14,8 +14,6 @@ function PullFtpFile {
|
|||
[int] $Buffersize = 1024
|
||||
)
|
||||
|
||||
|
||||
|
||||
$FTPRequest = [System.Net.FtpWebRequest]::Create($RemoteFile)
|
||||
$FTPRequest.Credentials = New-Object System.Net.NetworkCredential($Username,$Password)
|
||||
$FTPRequest.Method = [System.Net.WebRequestMethods+Ftp]::DownloadFile
|
||||
|
|
@ -24,7 +22,6 @@ function PullFtpFile {
|
|||
|
||||
$FTPResponse = $FTPRequest.GetResponse()
|
||||
$ResponseStream = $FTPResponse.GetResponseStream()
|
||||
|
||||
$LocalFileStream = New-Object IO.FileStream ($OutputPath,[IO.FileMode]::Create)
|
||||
if ($null -eq $LocalFileStream) {
|
||||
Write-Host "Write failed to file $OutputPath"
|
||||
|
|
@ -74,11 +71,13 @@ function ListFtpDirectory {
|
|||
}
|
||||
|
||||
$remoteDirectory = "ftp://bsb.hostedftp.com/~auspaynetftp/BSB/"
|
||||
$logPath = "./docs/source/samples/powershell/test/logs"
|
||||
|
||||
# Change local directory to suit
|
||||
$localDirectory = "./test"
|
||||
# Create required directory
|
||||
# Note that this assumes Windows environment
|
||||
$localDirectory = ".\docs\source\samples\powershell\test"
|
||||
if (!(Test-Path -Path $localDirectory)) {
|
||||
New-Item -Path $localDirectory
|
||||
$x = New-Item -Path $localDirectory
|
||||
}
|
||||
|
||||
Write-Host "First fetch the BSB files " -ForeGroundColor Green
|
||||
|
|
@ -86,21 +85,20 @@ Write-Host "First fetch the BSB files " -ForeGroundColor Green
|
|||
$fileList = ListFtpDirectory -Username "anonymous" -Password "anon@merebox.com" -RemoteFile $remoteDirectory
|
||||
Write-Host "File list size: $($fileList.count)" -ForegroundColor Gray
|
||||
|
||||
|
||||
Write-Host "Now iterate through the remote files and build remote martiLQ list " -ForeGroundColor Green
|
||||
|
||||
$oMarti = New-MartiDefinition
|
||||
$oMarti, $oConfig = New-MartiDefinition
|
||||
$oMarti.title = "Remote_BSB_data"
|
||||
$oMarti.description = "This definition covers the remote BSB data files `r downloaded from the Australian Payment Network"
|
||||
$oMarti.contactPoint = "meerkat@merebox.com"
|
||||
$oMarti.landingPage = "https://github.com/meerkat-manor/marti/blob/draft_specifications/docs/samples/powershell/Invoke-BSBSample.ps1"
|
||||
$oMarti.landingPage = "https://github.com/meerkat-manor/marti/blob/draft_specifications/docs/samples/powershell/Invoke-SampleGenerateBSB.ps1"
|
||||
$oMarti.theme = "payment"
|
||||
|
||||
ForEach ($item in $fileList) {
|
||||
if ($item -ne "" -and $item.startswith("BSBDirectory")) {
|
||||
PullFtpFile -Username "anonymous" -Password "anon@merebox.com" -RemoteFile ($remoteDirectory + $item) -OutputPath (Join-Path -Path $localDirectory -ChildPath $item)
|
||||
Write-Host "Add BSB $item file to Remote martiLQ metadata sample " -ForeGroundColor Yellow
|
||||
$oResource = New-MartiResource -SourcePath (Join-Path -Path $localDirectory -ChildPath $item) -UrlPath $remoteDirectory -LogPath ".\test\Logs" -ExtendAttributes
|
||||
$oResource = New-MartiResource -SourcePath (Join-Path -Path $localDirectory -ChildPath $item) -UrlPath $remoteDirectory -LogPath $logPath -ExtendAttributes
|
||||
if ($item.endswith(".txt")) {
|
||||
Set-AttributeValueNumber -Attributes $oResource.attributes -Key "header" -Category "dataset" -Function "count" -Value 1
|
||||
Set-AttributeValueNumber -Attributes $oResource.attributes -Key "footer" -Category "dataset" -Function "count" -Value 1
|
||||
|
|
@ -121,11 +119,11 @@ Write-Host "Remote martiLQ definition file is $fileJson " -ForeGroundColor Green
|
|||
|
||||
Write-Host "Now iterate through the local files and build martiLQ ZIP " -ForeGroundColor Green
|
||||
|
||||
$oMarti = New-MartiDefinition
|
||||
$oMarti, $oConfig = New-MartiDefinition
|
||||
$oMarti.title = "Zip_BSB_data"
|
||||
$oMarti.description = "This definition covers the ZIP BSB data files `r downloaded from the Australian Payment Network"
|
||||
$oMarti.contactPoint = "meerkat@merebox.com"
|
||||
$oMarti.landingPage = "https://github.com/meerkat-manor/marti/blob/draft_specifications/docs/samples/powershell/Invoke-BSBSample.ps1"
|
||||
$oMarti.landingPage = "https://github.com/meerkat-manor/marti/blob/draft_specifications/docs/samples/powershell/Invoke-SampleGenerateBSB.ps1"
|
||||
$oMarti.theme = "payment"
|
||||
|
||||
$zipFileName = "BSBDirectory.zip"
|
||||
|
|
@ -138,7 +136,7 @@ foreach($file in Get-ChildItem $localDirectory)
|
|||
if ($file.Name.startswith("BSBDirectory") -and !($file.Name.EndsWith(".zip")) -and !($file.Name.EndsWith(".7z")) ) {
|
||||
Write-Host "Add BSB file $file to ZIP martiLQ metadata sample " -ForeGroundColor Yellow
|
||||
Compress-Archive -Path $file.FullName -DestinationPath $zipFile -Update
|
||||
$oResource = New-MartiResource -SourcePath $file.FullName -UrlPath $localDirectory -LogPath ".\test\Logs" -ExtendAttributes
|
||||
$oResource = New-MartiResource -SourcePath $file.FullName -UrlPath $localDirectory -LogPath $logPath -ExtendAttributes
|
||||
if ($file.Extension -eq ".txt") {
|
||||
Set-AttributeValueNumber -Attributes $oResource.attributes -Key "header" -Category "dataset" -Function "count" -Value 1
|
||||
Set-AttributeValueNumber -Attributes $oResource.attributes -Key "footer" -Category "dataset" -Function "count" -Value 1
|
||||
|
|
@ -152,7 +150,7 @@ foreach($file in Get-ChildItem $localDirectory)
|
|||
$oMarti.resources += $oResource
|
||||
}
|
||||
}
|
||||
$oResource = New-MartiResource -SourcePath $zipFile -UrlPath $localDirectory -LogPath ".\test\Logs" -ExtendAttributes
|
||||
$oResource = New-MartiResource -SourcePath $zipFile -UrlPath $localDirectory -LogPath $logPath -ExtendAttributes
|
||||
Set-AttributeValueString -Attributes $oResource.attributes -Key "compression" -Category "format" -Function "algo" -Value "WINZIP"
|
||||
$oMarti.resources += $oResource
|
||||
|
||||
|
|
@ -164,11 +162,11 @@ Write-Host "ZIP martiLQ definition file is $fileJson " -ForeGroundColor Green
|
|||
|
||||
Write-Host "Now iterate through the local files with ZIP " -ForeGroundColor Green
|
||||
|
||||
$oMarti = New-MartiChildItem -SourceFolder $localDirectory -UrlPath "./test" -Filter "BSBDirectory*" -LogPath ".\test\Logs" -ExtendAttributes
|
||||
$oMarti = New-MartiChildItem -SourceFolder $localDirectory -UrlPath "./docs/source/samples/powershell/test" -Filter "BSBDirectory*" -LogPath $logPath -ExtendAttributes
|
||||
$oMarti.title = "Local_BSB_data"
|
||||
$oMarti.description = "This definition covers the local BSB data files `r downloaded from the Australian Payment Network"
|
||||
$oMarti.contactPoint = "meerkat@merebox.com"
|
||||
$oMarti.landingPage = "https://github.com/meerkat-manor/marti/blob/draft_specifications/docs/samples/powershell/Invoke-BSBSample.ps1"
|
||||
$oMarti.landingPage = "https://github.com/meerkat-manor/marti/blob/draft_specifications/docs/samples/powershell/Invoke-SampleGenerateBSB.ps1"
|
||||
$oMarti.theme = "payment"
|
||||
|
||||
$fileJson = Join-Path -Path $localDirectory -ChildPath "MartiBSBLocal.json"
|
||||
|
|
@ -178,11 +176,11 @@ Write-Host "Local martiLQ definition file is $fileJson " -ForeGroundColor Green
|
|||
|
||||
Write-Host "Now create an encrypted 7ZIP file with asymmetric password protection" -ForeGroundColor Green
|
||||
|
||||
$oMarti = New-MartiDefinition
|
||||
$oMarti, $oConfig = New-MartiDefinition
|
||||
$oMarti.title = "7ZIP_BSB_data"
|
||||
$oMarti.description = "This definition covers the 7ZIP BSB data files `r downloaded from the Australian Payment Network"
|
||||
$oMarti.contactPoint = "meerkat@merebox.com"
|
||||
$oMarti.landingPage = "https://github.com/meerkat-manor/marti/blob/draft_specifications/docs/samples/powershell/Invoke-BSBSample.ps1"
|
||||
$oMarti.landingPage = "https://github.com/meerkat-manor/marti/blob/draft_specifications/docs/samples/powershell/Invoke-SampleGenerateBSB.ps1"
|
||||
$oMarti.theme = "payment"
|
||||
|
||||
$zipFileName = "BSBDirectorySecure.7z"
|
||||
|
|
@ -199,7 +197,7 @@ foreach($file in Get-ChildItem $localDirectory)
|
|||
} else {
|
||||
Compress-7Zip -Path $file.FullName -ArchiveFileName $zipFile -Format SevenZip
|
||||
}
|
||||
$oResource = New-MartiResource -SourcePath $file.FullName -UrlPath $localDirectory -LogPath ".\test\Logs" -ExtendAttributes
|
||||
$oResource = New-MartiResource -SourcePath $file.FullName -UrlPath $localDirectory -LogPath $logPath -ExtendAttributes
|
||||
if ($file.Extension -eq ".txt") {
|
||||
Set-AttributeValueNumber -Attributes $oResource.attributes -Key "header" -Category "dataset" -Function "count" -Value 1
|
||||
Set-AttributeValueNumber -Attributes $oResource.attributes -Key "footer" -Category "dataset" -Function "count" -Value 1
|
||||
|
|
@ -216,13 +214,13 @@ foreach($file in Get-ChildItem $localDirectory)
|
|||
|
||||
$noticeFile = Join-Path -Path $localDirectory -ChildPath "README.txt"
|
||||
Set-Content -Path $noticeFile -Value "Generated by martiLQ Samples"
|
||||
$oResource = New-MartiResource -SourcePath $noticeFile -UrlPath $localDirectory -LogPath ".\test\Logs"
|
||||
$oResource = New-MartiResource -SourcePath $noticeFile -UrlPath $localDirectory -LogPath $logPath
|
||||
$oMarti.resources += $oResource
|
||||
|
||||
$secret = "change_me_to_secure"
|
||||
Compress-7Zip -Path $noticeFile -ArchiveFileName $zipFile -Append -Password $secret -EncryptFilenames
|
||||
|
||||
$oResource = New-MartiResource -SourcePath $zipFile -UrlPath $localDirectory -LogPath ".\test\Logs" -ExtendAttributes
|
||||
$oResource = New-MartiResource -SourcePath $zipFile -UrlPath $localDirectory -LogPath $logPath -ExtendAttributes
|
||||
$oResource.compression = "7ZIP"
|
||||
$oResource.encryption = New-Encryption -Algorithm "Passphrase" -Value $secret
|
||||
Set-AttributeValueString -Attributes $oResource.attributes -Key "compression" -Category "format" -Function "algo" -Value "7ZIP"
|
||||
|
|
|
|||
|
|
@ -96,6 +96,8 @@ Write-Host "Please execute in the same directory as script" -ForeGroundColor Yel
|
|||
|
||||
$recipientKey = "CN=PeterDocs"
|
||||
$remoteDirectory = "ftp://bsb.hostedftp.com/~auspaynetftp/BSB/"
|
||||
$logPath = "./docs/source/samples/powershell/test/logs"
|
||||
|
||||
|
||||
# Change local directory to suit
|
||||
$localDirectory = "./test"
|
||||
|
|
@ -110,11 +112,11 @@ Write-Host "File list size: $($fileList.count)" -ForegroundColor Gray
|
|||
|
||||
Write-Host "Now create an encrypted 7ZIP file with password " -ForeGroundColor Green
|
||||
|
||||
$oMarti = New-MartiDefinition
|
||||
$oMarti, $oConfig = New-MartiDefinition
|
||||
$oMarti.title = "7ZIP_BSB_data"
|
||||
$oMarti.description = "This definition covers the 7ZIP BSB data files `r downloaded from the Australian Payment Network"
|
||||
$oMarti.contactPoint = "meerkat@merebox.com"
|
||||
$oMarti.landingPage = "https://github.com/meerkat-manor/marti/blob/draft_specifications/docs/samples/powershell/Invoke-BSBSample.ps1"
|
||||
$oMarti.landingPage = "https://github.com/meerkat-manor/marti/blob/draft_specifications/docs/samples/powershell/Invoke-SampleGenerateBSBSecure.ps1"
|
||||
$oMarti.theme = "payment"
|
||||
|
||||
$zipFileName = "BSBDirectoryPKI.7z"
|
||||
|
|
@ -131,7 +133,7 @@ foreach($file in Get-ChildItem $localDirectory)
|
|||
} else {
|
||||
Compress-7Zip -Path $file.FullName -ArchiveFileName $zipFile -Format SevenZip
|
||||
}
|
||||
$oResource = New-MartiResource -SourcePath $file.FullName -UrlPath $localDirectory -LogPath ".\test\Logs" -ExtendAttributes
|
||||
$oResource = New-MartiResource -SourcePath $file.FullName -UrlPath $localDirectory -LogPath $logPath -ExtendAttributes
|
||||
if ($file.Extension -eq ".txt") {
|
||||
Set-AttributeValueNumber -Attributes $oResource.attributes -Key "header" -Category "dataset" -Function "count" -Value 1
|
||||
Set-AttributeValueNumber -Attributes $oResource.attributes -Key "footer" -Category "dataset" -Function "count" -Value 1
|
||||
|
|
@ -148,7 +150,7 @@ foreach($file in Get-ChildItem $localDirectory)
|
|||
|
||||
$noticeFile = Join-Path -Path $localDirectory -ChildPath "README.txt"
|
||||
Set-Content -Path $noticeFile -Value "Generated by martiLQ Samples"
|
||||
$oResource = New-MartiResource -SourcePath $noticeFile -UrlPath $localDirectory -LogPath ".\test\Logs"
|
||||
$oResource = New-MartiResource -SourcePath $noticeFile -UrlPath $localDirectory -LogPath $logPath
|
||||
$oMarti.resources += $oResource
|
||||
|
||||
$secret = New-RandomPassword -Length 80
|
||||
|
|
@ -156,7 +158,7 @@ $encryptedSecret = Protect-CmsMessage -To $recipientKey -Content $secret
|
|||
|
||||
Compress-7Zip -Path $noticeFile -ArchiveFileName $zipFile -Append -Password $secret -EncryptFilenames
|
||||
|
||||
$oResource = New-MartiResource -SourcePath $zipFile -UrlPath $localDirectory -LogPath ".\test\Logs" -ExtendAttributes
|
||||
$oResource = New-MartiResource -SourcePath $zipFile -UrlPath $localDirectory -LogPath $logPath -ExtendAttributes
|
||||
$oResource.compression = "7ZIP"
|
||||
$oResource.encryption = New-Encryption -Algorithm "PKI" -Value $($encryptedSecret)
|
||||
|
||||
|
|
|
|||
|
|
@ -0,0 +1,20 @@
|
|||
|
||||
. .\source\powershell\MartiLQ.ps1
|
||||
|
||||
try {
|
||||
|
||||
$describe = "Generate martiLQ definition for DOCS folder "
|
||||
Write-Host $describe
|
||||
$oMarti = New-MartiChildItem -SourceFolder "./docs/source" -UrlPath "/docs" -Filter "*.md" -LogPath "./docs/source/samples/powershell/test/Logs" -ExtendAttributes:$true
|
||||
$oMarti.description = $describe
|
||||
|
||||
$jsonFile = "./docs/source/samples/powershell/test/martilq_docs.json"
|
||||
$x = ConvertTo-Json -InputObject $oMarti -Depth 6
|
||||
Set-Content -Path $jsonFile -Value $x
|
||||
|
||||
Write-Host "martiLQ definition written to '$jsonFile' "
|
||||
|
||||
}
|
||||
catch {
|
||||
throw
|
||||
}
|
||||
|
|
@ -0,0 +1,25 @@
|
|||
|
||||
# Powershell samples
|
||||
|
||||
To execute the Python scripts, please invoke from the root Marti directory and not
|
||||
with the current directory set to ``./docs/source/samples/powershell``
|
||||
|
||||
```powershell
|
||||
|
||||
# To seed the test data execute these two Powershell scripts
|
||||
./docs/source/samples/powershell/Invoke-SampleGenerateBsb.ps1
|
||||
./docs/source/samples/powershell/Invoke-SampleGenerateBsbSecure.ps1
|
||||
# To see
|
||||
./docs/source/samples/powershell/Invoke-SampleGenerateDocs.ps1
|
||||
```
|
||||
|
||||
## BSB samples
|
||||
|
||||
The BSB PowerShell scripts demonstrate the retrieval of Internet resources
|
||||
and building a **martiLQ** definition file.
|
||||
|
||||
## Docs sample
|
||||
|
||||
The ``Invoke-SampleGenerateDocs.ps1`` script demonstrates a local directory
|
||||
being processed and for a **martiLQ** definition file. The directory is the
|
||||
documentation Markdown files.
|
||||
|
|
@ -0,0 +1,18 @@
|
|||
|
||||
|
||||
|
||||
To execute the Python scripts, please invoke from the root Marti directory and not from
|
||||
with the current directory set to ``./docs/source/samples/python``
|
||||
|
||||
``
|
||||
# Fetch the data and generate the Json
|
||||
./docs/source/samples/python/SampleGenerateFtpBsb.py
|
||||
./docs/source/samples/python/SampleGenerateHttpBsb.py
|
||||
|
||||
# Uisng the Json fetch the files and reconcile
|
||||
./docs/source/samples/python/SampleJsonFtpBsb.py
|
||||
./docs/source/samples/python/SampleJsonHttpBsb.py
|
||||
|
||||
``
|
||||
|
||||
|
||||
|
|
@ -7,11 +7,11 @@ import csv
|
|||
import zipfile
|
||||
|
||||
|
||||
sys.path.insert(0, "../../../../source/python/client")
|
||||
sys.path.insert(0, "./source/python/client")
|
||||
from martiLQ import *
|
||||
|
||||
ftpFetch = True
|
||||
os.environ["MARTILQ_LOGPATH"] = "./test/logs"
|
||||
os.environ["MARTILQ_LOGPATH"] = "./docs/source/samples/python/test/logs"
|
||||
|
||||
def ftpList(host, path):
|
||||
|
||||
|
|
@ -53,11 +53,11 @@ remote_dir = "/~auspaynetftp/BSB/"
|
|||
print("Fetch sample file list")
|
||||
files = ftpList(remote_host, remote_dir)
|
||||
|
||||
test_dir = "./test/ftp"
|
||||
test_dir = "./docs/source/samples/python/test/fetch_ftp"
|
||||
if not os.path.exists(test_dir):
|
||||
os.mkdir(test_dir)
|
||||
|
||||
print("Fetch sample files")
|
||||
print("Fetch sample files via FTP")
|
||||
for file_name in files:
|
||||
if file_name.startswith("BSBDirectory"):
|
||||
if file_name.endswith(".csv") | file_name.endswith(".txt"):
|
||||
|
|
@ -123,8 +123,6 @@ jsonFile.write(json.dumps(oMarti, indent=5))
|
|||
jsonFile.close()
|
||||
print("ZIP sample JSON written: martilq_bsb_zip.json")
|
||||
|
||||
|
||||
|
||||
print("Sample completed: SampleGenerateFtpBsb.py")
|
||||
|
||||
lqresults, testError = mlq.TestMartiDefinition(os.path.join(test_dir, "martilq_bsb.json"))
|
||||
|
|
@ -136,6 +134,8 @@ with testfile:
|
|||
|
||||
if testError:
|
||||
print("MISMATCH DETECTED")
|
||||
else:
|
||||
print("MATCHED")
|
||||
|
||||
print("Test completed: SampleGenerateFtpBsb.py")
|
||||
|
||||
|
|
|
|||
|
|
@ -9,18 +9,18 @@ import zipfile
|
|||
import datetime
|
||||
import time
|
||||
|
||||
sys.path.insert(0, "../../../../source/python/client")
|
||||
sys.path.insert(0, "./source/python/client")
|
||||
from martiLQ import *
|
||||
|
||||
httpFetch = True
|
||||
os.environ["MARTILQ_LOGPATH"] = "./test/logs"
|
||||
os.environ["MARTILQ_LOGPATH"] = "./docs/source/samples/python/test/logs"
|
||||
|
||||
|
||||
def HttpList(remote_url):
|
||||
|
||||
files = []
|
||||
|
||||
with open("listfiles_bsb.txt", "r") as f:
|
||||
with open("./docs/source/samples/python/listfiles_bsb_http.txt", "r") as f:
|
||||
files = f.read().splitlines()
|
||||
|
||||
return files
|
||||
|
|
@ -31,15 +31,16 @@ remote_url = "http://apnedata.merebox.com.s3.ap-southeast-2.amazonaws.com/au/bsb
|
|||
print("Fetch sample file list")
|
||||
files = HttpList(remote_url)
|
||||
|
||||
test_dir = "./test/http"
|
||||
test_dir = "./docs/source/samples/python/test/fetch_http"
|
||||
if not os.path.exists(test_dir):
|
||||
os.mkdir(test_dir)
|
||||
|
||||
if httpFetch:
|
||||
print("Fetch sample files")
|
||||
print("Fetch sample files via HTTP")
|
||||
for file_name in files:
|
||||
if file_name.startswith("BSBDirectory"):
|
||||
if file_name.endswith(".csv") | file_name.endswith(".txt"):
|
||||
try:
|
||||
with urllib.request.urlopen(remote_url + file_name) as resp:
|
||||
last_modified = resp.info()["Last-Modified"]
|
||||
dt_obj = datetime.datetime.strptime(last_modified, '%a, %d %b %Y %H:%M:%S %Z')
|
||||
|
|
@ -50,6 +51,8 @@ if httpFetch:
|
|||
|
||||
modTime = time.mktime(dt_obj.timetuple())
|
||||
os.utime(data_file_name, (modTime, modTime))
|
||||
except:
|
||||
print("error with fetching "+remote_url + file_name)
|
||||
|
||||
print("Creating martiLQ definition")
|
||||
mlq = martiLQ()
|
||||
|
|
@ -61,7 +64,7 @@ for file_name in files:
|
|||
oResource = mlq.NewMartiLQResource(os.path.join(test_dir, file_name), "", False, True)
|
||||
oMarti["resources"].append(oResource)
|
||||
|
||||
mlq.CloseLog()
|
||||
mlq.Close()
|
||||
|
||||
|
||||
print("Save martiLQ definition")
|
||||
|
|
@ -90,11 +93,10 @@ with zipfile.ZipFile(os.path.join(test_dir, zipFileName), "w", compression=zipfi
|
|||
|
||||
oResource = mlq.NewMartiLQResource(os.path.join(test_dir, zipFileName), "", False, True)
|
||||
oResource["url"] = test_dir + zipFileName
|
||||
mlq.SetAttributeValueString(Attributes=oResource["attributes"], Key="compression", Category="format", Function="algo", Value="WINZIP")
|
||||
mlq.SetAttributeValueNumber(Attributes=oResource["attributes"], Key="files", Category="dataset", Function="count", Value=fileZipCount)
|
||||
mResource.SetAttributeValueNumber(oResource, Key="files", Category="dataset", Function="count", Value=fileZipCount)
|
||||
oMarti["resources"].append(oResource)
|
||||
|
||||
mlq.CloseLog()
|
||||
mlq.Close()
|
||||
|
||||
print("Save martiLQ ZIP definition")
|
||||
jsonFile = open(os.path.join(test_dir, "MartiLQ_BSBZip.json"), "w")
|
||||
|
|
@ -104,7 +106,7 @@ print("ZIP sample JSON written: MartiLQ_BSBZip.json")
|
|||
|
||||
|
||||
|
||||
print("Sample completed: SampleGenerateBsb.py")
|
||||
print("Sample completed: SampleGenerateHttpBsb.py")
|
||||
|
||||
lqresults, testError = mlq.TestMartiDefinition(os.path.join(test_dir, "BSBDirectoryPlain.json"))
|
||||
|
||||
|
|
@ -115,6 +117,8 @@ with testfile:
|
|||
|
||||
if testError:
|
||||
print("MISMATCH DETECTED")
|
||||
else:
|
||||
print("MATCHED")
|
||||
|
||||
print("Test completed: SampleGenerateFtpBsb.py")
|
||||
print("Test completed: SampleGenerateHttpBsb.py")
|
||||
|
||||
|
|
|
|||
|
|
@ -3,24 +3,24 @@ import os
|
|||
import sys
|
||||
import csv
|
||||
|
||||
sys.path.insert(0, "../../../../source/python/client")
|
||||
sys.path.insert(0, "./source/python/client")
|
||||
from martiLQ import *
|
||||
|
||||
test_directory = "./test/fetch_ftp"
|
||||
test_directory = "./docs/source/samples/python/test/fetch_ftp"
|
||||
os.environ["MARTILQ_LOGPATH"] = os.path.join(test_directory, "logs")
|
||||
|
||||
if not os.path.exists("./test"):
|
||||
os.mkdir("./test")
|
||||
if not os.path.exists("./docs/source/samples/python/test"):
|
||||
os.mkdir("./docs/source/samples/python/test")
|
||||
|
||||
if not os.path.exists(test_directory):
|
||||
os.mkdir(test_directory)
|
||||
|
||||
print("Creating martiLQ definition")
|
||||
mlq = martiLQ()
|
||||
mlq.LoadConfig()
|
||||
mlq.LoadConfig(ConfigPath=None)
|
||||
print("Loading definition json")
|
||||
mlq.Load("../BSBDirectoryFtp.json")
|
||||
print("Fetching files")
|
||||
mlq.Load("./docs/source/samples/json/BSBDirectoryFtp.json")
|
||||
print("Fetching files based on URL (ftp)")
|
||||
fetched_files, fetch_error = mlq.Fetch(test_directory)
|
||||
|
||||
if len(fetched_files) < 0:
|
||||
|
|
@ -38,9 +38,9 @@ for full_fileName in fetched_files:
|
|||
oResource = mlq.NewMartiLQResource(full_fileName, "", False, True)
|
||||
oMarti["resources"].append(oResource)
|
||||
print("Perform validation test")
|
||||
lqresults, testError = mlq.TestMartiDefinition("../BSBDirectoryFtp.json")
|
||||
lqresults, testError = mlq.TestMartiDefinition("./docs/source/samples/json/BSBDirectoryFtp.json")
|
||||
|
||||
testfile = open("./test/LoadQualityTest_Ftp.csv", "w+", newline ="")
|
||||
testfile = open(os.path.join(test_directory, "LoadQualityTest_Ftp.csv"), "w+", newline ="")
|
||||
with testfile:
|
||||
lqwriter = csv.writer(testfile)
|
||||
lqwriter.writerows(lqresults)
|
||||
|
|
@ -50,7 +50,6 @@ if testError > 0:
|
|||
else:
|
||||
print("RECONCILED")
|
||||
|
||||
mlq.CloseLog()
|
||||
|
||||
print("Test completed: SampleFetchFtpBsb.py")
|
||||
mlq.Close()
|
||||
|
||||
print("Sample completed: SampleJsonFtpBsb.py")
|
||||
|
|
@ -3,24 +3,24 @@ import os
|
|||
import sys
|
||||
import csv
|
||||
|
||||
sys.path.insert(0, "../../../../source/python/client")
|
||||
sys.path.insert(0, "./source/python/client")
|
||||
from martiLQ import *
|
||||
|
||||
test_directory = "./test/fetch_http"
|
||||
test_directory = "./docs/source/samples/python/test/fetch_http"
|
||||
os.environ["MARTILQ_LOGPATH"] = os.path.join(test_directory, "logs")
|
||||
|
||||
if not os.path.exists("./test"):
|
||||
os.mkdir("./test")
|
||||
if not os.path.exists("./docs/source/samples/python/test"):
|
||||
os.mkdir("./docs/source/samples/python/test")
|
||||
|
||||
if not os.path.exists(test_directory):
|
||||
os.mkdir(test_directory)
|
||||
|
||||
print("Creating martiLQ definition")
|
||||
mlq = martiLQ()
|
||||
mlq.LoadConfig()
|
||||
mlq.LoadConfig(ConfigPath=None)
|
||||
print("Loading definition json")
|
||||
mlq.Load("../BSBDirectoryHttp.json")
|
||||
print("Fetching files")
|
||||
mlq.Load("./docs/source/samples/json/BSBDirectoryHttp.json")
|
||||
print("Fetching files based on URL (ftp)")
|
||||
fetched_files, fetch_error = mlq.Fetch(test_directory)
|
||||
|
||||
if len(fetched_files) < 0:
|
||||
|
|
@ -38,7 +38,7 @@ for full_fileName in fetched_files:
|
|||
oResource = mlq.NewMartiLQResource(full_fileName, "", False, True)
|
||||
oMarti["resources"].append(oResource)
|
||||
print("Perform validation test")
|
||||
lqresults, testError = mlq.TestMartiDefinition("../BSBDirectoryHttp.json")
|
||||
lqresults, testError = mlq.TestMartiDefinition("./docs/source/samples/json/BSBDirectoryHttp.json")
|
||||
|
||||
testfile = open("./test/LoadQualityTest_Http.csv", "w+", newline ="")
|
||||
with testfile:
|
||||
|
|
@ -50,7 +50,6 @@ if testError > 0:
|
|||
else:
|
||||
print("RECONCILED")
|
||||
|
||||
mlq.CloseLog()
|
||||
|
||||
print("Test completed: SampleFetchHttpBsb.py")
|
||||
mlq.Close()
|
||||
|
||||
print("Sample completed: SampleJsonHttpBsb.py")
|
||||
|
|
@ -0,0 +1,16 @@
|
|||
BSBDirectoryOct21-307.csv
|
||||
BSBDirectorySep21-306.csv
|
||||
BSBDirectoryAug21-305.csv
|
||||
BSBDirectoryJul21-304.csv
|
||||
BSBDirectoryJun21-303.csv
|
||||
BSBDirectoryMay21-302.csv
|
||||
BSBDirectoryApr21-301.csv
|
||||
BSBDirectoryMar21-300.csv
|
||||
BSBDirectoryOct21-307.txt
|
||||
BSBDirectorySep21-306.txt
|
||||
BSBDirectoryAug21-305.txt
|
||||
BSBDirectoryJul21-304.txt
|
||||
BSBDirectoryJun21-303.txt
|
||||
BSBDirectoryMay21-302.txt
|
||||
BSBDirectoryApr21-301.txt
|
||||
BSBDirectoryMar21-300.txt
|
||||
|
|
@ -5,12 +5,12 @@ import (
|
|||
"os"
|
||||
)
|
||||
|
||||
func TestMarti_JsonSave(t *testing.T) {
|
||||
func TestMartiLQ_JsonSave(t *testing.T) {
|
||||
m:= NewMarti()
|
||||
m.Save("../test/basic_test.json")
|
||||
}
|
||||
|
||||
func TestMarti_ResourceAdd(t *testing.T) {
|
||||
func TestMartiLQ_ResourceAdd(t *testing.T) {
|
||||
m := NewMarti()
|
||||
r := NewResource(m.config)
|
||||
r.Title = "Title text"
|
||||
|
|
@ -24,7 +24,7 @@ func TestMarti_ResourceAdd(t *testing.T) {
|
|||
m.Save("../test/test_addresource.json")
|
||||
}
|
||||
|
||||
func TestMarti_ResourceExpire(t *testing.T) {
|
||||
func TestMartiLQ_ResourceExpire(t *testing.T) {
|
||||
m := NewMarti()
|
||||
m.LoadConfig("../../../../martilq.ini")
|
||||
r := NewResource(m.config)
|
||||
|
|
@ -40,7 +40,7 @@ func TestMarti_ResourceExpire(t *testing.T) {
|
|||
}
|
||||
|
||||
|
||||
func TestMarti_DirectoryA(t *testing.T) {
|
||||
func TestMartiLQ_DirectoryA(t *testing.T) {
|
||||
|
||||
currentDirectory, _ := os.Getwd()
|
||||
SourcePath := currentDirectory
|
||||
|
|
@ -50,7 +50,7 @@ func TestMarti_DirectoryA(t *testing.T) {
|
|||
|
||||
}
|
||||
|
||||
func TestMarti_DirectoryB(t *testing.T) {
|
||||
func TestMartiLQ_DirectoryB(t *testing.T) {
|
||||
|
||||
currentDirectory, _ := os.Getwd()
|
||||
SourcePath := currentDirectory
|
||||
|
|
|
|||
|
|
@ -1,6 +1,6 @@
|
|||
FROM busybox
|
||||
|
||||
LABEL version="202111B"
|
||||
LABEL version="202111C"
|
||||
LABEL description="martiLQ Go web UI by Meerkat Manor "
|
||||
LABEL authors="meerkat@merebox.com"
|
||||
|
||||
|
|
@ -12,4 +12,4 @@ COPY ./src/data /martilq/data
|
|||
|
||||
EXPOSE 8080
|
||||
|
||||
ENTRYPOINT exec /martilq/main -p 8080 -static /martilq/static -data /martilq/data/ -docs /martilq/docs -template /martilq/template
|
||||
ENTRYPOINT exec /martilq/main -bind :8080 -static /martilq/static -data /martilq/data/ -docs /martilq/docs -template /martilq/template
|
||||
|
|
|
|||
|
|
@ -11,5 +11,5 @@ services:
|
|||
- 8080:8080
|
||||
|
||||
volumes:
|
||||
- ./src/data:/home/data
|
||||
- ./src/docs:/home/docs
|
||||
- ./src/data:/martilq/data
|
||||
- ./src/docs:/martilq/docs
|
||||
|
|
|
|||
|
|
@ -59,7 +59,7 @@
|
|||
"labels": {
|
||||
"leftLabel": "Branch",
|
||||
"rightLabel": "",
|
||||
"heading": ""
|
||||
"heading": "Branch"
|
||||
}
|
||||
},
|
||||
{
|
||||
|
|
|
|||
File diff suppressed because it is too large
Load Diff
|
|
@ -0,0 +1,5 @@
|
|||
module merebox.com/martilq
|
||||
|
||||
go 1.16
|
||||
|
||||
require github.com/russross/blackfriday v1.6.0
|
||||
|
|
@ -0,0 +1,2 @@
|
|||
github.com/russross/blackfriday v1.6.0 h1:KqfZb0pUVN2lYqZUYRddxF4OR8ZMURnJIG5Y3VRLtww=
|
||||
github.com/russross/blackfriday v1.6.0/go.mod h1:ti0ldHuxg49ri4ksnFxlkCfN+hvslNlmVHqNRXXJNAY=
|
||||
|
|
@ -9,10 +9,17 @@ import (
|
|||
"io/ioutil"
|
||||
"os"
|
||||
"encoding/json"
|
||||
|
||||
"errors"
|
||||
"html/template"
|
||||
"github.com/russross/blackfriday"
|
||||
)
|
||||
|
||||
|
||||
|
||||
|
||||
func main() {
|
||||
port := flag.String("p", "8080", "Http listen port")
|
||||
bind := flag.String("bind", ":8080", "Bind Http listen to address and port, e.g. localhost:8080 or justy simply :8080")
|
||||
staticDirectory := flag.String("static", "static", "Static directory content")
|
||||
docsDirectory := flag.String("docs", "", "Document directory content")
|
||||
dataDirectory := flag.String("data", "", "Data directory content")
|
||||
|
|
@ -49,7 +56,43 @@ func main() {
|
|||
http.ServeFile(res, req, safePath)
|
||||
})
|
||||
|
||||
|
||||
http.HandleFunc("/docs/", func( res http.ResponseWriter, req *http.Request ) {
|
||||
|
||||
if !strings.HasSuffix(req.URL.Path, ".md") {
|
||||
//http.Handler.ServeHTTP(http.Handler, res, req)
|
||||
return
|
||||
}
|
||||
|
||||
var pathErr *os.PathError
|
||||
input, err := ioutil.ReadFile("." + req.URL.Path)
|
||||
if errors.As(err, &pathErr) {
|
||||
http.Error(res, http.StatusText(http.StatusNotFound)+": "+req.URL.Path, http.StatusNotFound)
|
||||
log.Printf("file not found: %s", err)
|
||||
return
|
||||
}
|
||||
|
||||
if err != nil {
|
||||
http.Error(res, "Internal Server Error: "+err.Error(), 500)
|
||||
log.Printf("Couldn't read path %s: %v (%T)", req.URL.Path, err, err)
|
||||
return
|
||||
}
|
||||
|
||||
output := blackfriday.MarkdownCommon(input)
|
||||
|
||||
res.Header().Set("contentType", "text/html; charset=utf-8")
|
||||
|
||||
outputTemplate.Execute(res, struct {
|
||||
Path string
|
||||
Body template.HTML
|
||||
}{
|
||||
Path: req.URL.Path,
|
||||
Body: template.HTML(string(output)),
|
||||
})
|
||||
|
||||
})
|
||||
|
||||
http.HandleFunc("/docsx/", func( res http.ResponseWriter, req *http.Request ) {
|
||||
localPath := ""
|
||||
if (*docsDirectory == "") {
|
||||
temp := "../../.."
|
||||
|
|
@ -84,8 +127,8 @@ func main() {
|
|||
fileServer := http.FileServer(FileSystem{http.Dir(*staticDirectory)})
|
||||
http.Handle("/", fileServer)
|
||||
|
||||
log.Printf("Serving on HTTP port: %s\n", *port)
|
||||
log.Fatal(http.ListenAndServe(":"+*port, nil))
|
||||
log.Printf("Serving HTTP on address and port: %s\n", *bind)
|
||||
log.Fatal(http.ListenAndServe(*bind, nil))
|
||||
}
|
||||
|
||||
|
||||
|
|
@ -194,3 +237,19 @@ func apiHandlerView(res http.ResponseWriter, req *http.Request) {
|
|||
}
|
||||
|
||||
}
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
var outputTemplate = template.Must(template.New("base").Parse(`
|
||||
<html>
|
||||
<head>
|
||||
<title>{{ .Path }}</title>
|
||||
</head>
|
||||
<body>
|
||||
{{ .Body }}
|
||||
</body>
|
||||
</html>
|
||||
`))
|
||||
|
||||
|
|
|
|||
|
|
@ -42,7 +42,7 @@
|
|||
<h2>MartiLQ definition</h2>
|
||||
<div class="table-responsive">
|
||||
|
||||
<p>The following <string>martiLQ</string> definitions are provided as samples</p>
|
||||
<p>The following <strong>martiLQ</strong> definitions are provided as samples</p>
|
||||
<p>The <a href="https://github.com/meerkat-manor/marti">server code</a> for this is available and the code can be changed to reflect your actual or sample definitions</p>
|
||||
|
||||
<br>
|
||||
|
|
|
|||
|
|
@ -62,6 +62,9 @@ function fetchData(dataFile) {
|
|||
});
|
||||
}
|
||||
// Use the template
|
||||
if (template == "") {
|
||||
template = "template/martilq_no_struct.must"
|
||||
}
|
||||
fetchMust(template);
|
||||
|
||||
});
|
||||
|
|
|
|||
|
|
@ -10,7 +10,7 @@ function New-MartiDefinition
|
|||
|
||||
$oSoftware = [PSCustomObject]@{
|
||||
extension = "software"
|
||||
softwareName = "MartiReference"
|
||||
softwareName = Get-SoftwareName
|
||||
author = "Meerkat@merebox.com"
|
||||
version = "$script:SoftwareVersion"
|
||||
}
|
||||
|
|
@ -21,7 +21,13 @@ function New-MartiDefinition
|
|||
url = ""
|
||||
}
|
||||
|
||||
|
||||
$oConfig = Get-DefaultConfiguration
|
||||
if ( $nulll -eq $oConfig.publisher -or $oConfig.publisher -eq "") {
|
||||
$publisher = [System.Security.Principal.WindowsIdentity]::GetCurrent().Name
|
||||
} else {
|
||||
$publisher = $oConfig.publisher
|
||||
}
|
||||
|
||||
[System.Collections.ArrayList]$lcustom = @()
|
||||
$lcustom += $oSoftware
|
||||
|
|
@ -29,7 +35,7 @@ function New-MartiDefinition
|
|||
|
||||
[System.Collections.ArrayList]$lresource = @()
|
||||
|
||||
$expires = (Get-Date).AddYears(7)
|
||||
$expires = Set-DefaultExpiryDate -DocumentDate (Get-Date) -Configuration $oConfig
|
||||
|
||||
$oMarti = [PSCustomObject]@{
|
||||
contentType = "application/vnd.martilq.json"
|
||||
|
|
@ -37,26 +43,26 @@ function New-MartiDefinition
|
|||
uid = (New-Guid).ToString()
|
||||
|
||||
description = ""
|
||||
issued = Get-Date -f "yyyy-MM-ddTHH:mm:ss"
|
||||
modified = Get-Date -f "yyyy-MM-ddTHH:mm:ss"
|
||||
expires = $expires -f "yyyy-MM-ddTHH:mm:ss"
|
||||
tags = @( "document", "marti")
|
||||
issued = Get-Date -f $oConfig.dateTimeFormat
|
||||
modified = Get-Date -f $oConfig.dateTimeFormat
|
||||
expires = $expires.Tostring($oConfig.dateTimeFormat)
|
||||
tags = $oConfig.tags
|
||||
publisher = $publisher
|
||||
contactPoint = ""
|
||||
accessLevel = "Confidential"
|
||||
rights = "Restricted"
|
||||
license = ""
|
||||
state = "active"
|
||||
batch = 1.0
|
||||
contactPoint = $oConfig.contactPoint
|
||||
accessLevel = $oConfig.accessLevel
|
||||
rights = $oConfig.rights
|
||||
license = $oConfig.license
|
||||
state = $oConfig.state
|
||||
batch = $oConfig.batch
|
||||
describedBy = ""
|
||||
landingPage = ""
|
||||
theme =""
|
||||
theme =$oConfig.theme
|
||||
|
||||
resources = $lresource
|
||||
custom = $lCustom
|
||||
}
|
||||
|
||||
return $oMarti
|
||||
return $oMarti, $oConfig
|
||||
}
|
||||
|
||||
|
||||
|
|
@ -199,7 +205,7 @@ Param(
|
|||
|
||||
$oCkan = ConvertFrom-Json -InputObject $InputObject
|
||||
|
||||
$oMarti = New-MartiDefinition
|
||||
$oMarti, $oConfig = New-MartiDefinition
|
||||
|
||||
$oMarti.title = "Conversion from CKAN"
|
||||
$oMarti.state = $oCkan.result.state
|
||||
|
|
@ -278,12 +284,12 @@ Param(
|
|||
Write-Log "Parameter: Filter Value: $Filter "
|
||||
Write-Log ""
|
||||
|
||||
$marti_mri = $global:default_metaFile
|
||||
$martilq_mri = $global:default_metaFile
|
||||
|
||||
$oMarti = New-MartiDefinition -SourceFolder $SourceFolder -Filter $Filter -LogPath $LogPath
|
||||
$oMarti, $oConfig = New-MartiDefinition -SourceFolder $SourceFolder -Filter $Filter -LogPath $LogPath
|
||||
$oMarti.description = "Sample execution"
|
||||
|
||||
$fullMetadatName = Join-Path -Path (Split-Path -Path $ArchiveFile -Parent) -ChildPath $marti_mri
|
||||
$fullMetadatName = Join-Path -Path (Split-Path -Path $ArchiveFile -Parent) -ChildPath $martilq_mri
|
||||
$x = ConvertTo-Json -InputObject $oMarti
|
||||
Add-Content -Path $fullMetadatName -Value $x
|
||||
|
||||
|
|
|
|||
|
|
@ -2,5 +2,61 @@
|
|||
|
||||
|
||||
$script:SoftwareVersion = "0.0.1"
|
||||
$global:default_metaFile = "##marti##.json"
|
||||
$global:default_metaFile = "##martilq##.json"
|
||||
|
||||
|
||||
function Get-DefaultConfiguration {
|
||||
|
||||
$oConfiguration = [PSCustomObject]@{
|
||||
softwareName = Get-SoftwareName
|
||||
softwareAuthor = "Meerkat@merebox.com"
|
||||
softwareVersion = $script:SoftwareVersion
|
||||
|
||||
logPath = "./logs/"
|
||||
dateFormat = "yyyy-MM-dd"
|
||||
dateTimeFormat = "yyyy-MM-ddTHH:mm:ss"
|
||||
dataPath = ""
|
||||
tempPath = ""
|
||||
|
||||
tags = @( "default", "martiLQ")
|
||||
publisher = ""
|
||||
contactPoint = ""
|
||||
license = ""
|
||||
accessLevel = "Confidential"
|
||||
rights = "Restricted"
|
||||
batch = 1.0000
|
||||
batchInc = 0.0001
|
||||
theme = ""
|
||||
|
||||
author = $null
|
||||
title = "{{documentName.ext}}"
|
||||
state = "active"
|
||||
expires = "m:10:0:0"
|
||||
version = "1.0"
|
||||
urlPrefix = ""
|
||||
encoding = ""
|
||||
compression = ""
|
||||
encryption = ""
|
||||
describedBy = ""
|
||||
landingPage = ""
|
||||
|
||||
hashAlgorithm = "SHA256"
|
||||
signKey_File = $null
|
||||
signKey_Password = $null
|
||||
|
||||
proxy = $null
|
||||
proxy_User = $null
|
||||
proxy_Credential = $null
|
||||
|
||||
loaded = $false
|
||||
}
|
||||
|
||||
#self._Log = mLogging()
|
||||
#self._Log.SetConfig(self._oConfiguration["logPath"], self.GetSoftwareName())
|
||||
|
||||
return $oConfiguration
|
||||
}
|
||||
|
||||
function Get-Title {
|
||||
|
||||
}
|
||||
|
|
|
|||
|
|
@ -46,6 +46,7 @@ Param(
|
|||
[String] $UrlPath = "",
|
||||
[switch] $ExcludeHash,
|
||||
[switch] $ExtendAttributes,
|
||||
[PSCustomObject] $Configuration = $null,
|
||||
[String] $LogPath
|
||||
|
||||
)
|
||||
|
|
@ -61,6 +62,9 @@ Param(
|
|||
Write-Log "Parameter: ExcludeHash Value: $ExcludeHash "
|
||||
Write-Log ""
|
||||
|
||||
if ($null -eq $Configuration) {
|
||||
$Configuration = Get-DefaultConfiguration
|
||||
}
|
||||
|
||||
if (Test-Path -Path $SourcePath -PathType Leaf) {
|
||||
|
||||
|
|
@ -71,21 +75,22 @@ Param(
|
|||
if ($ExcludeHash) {
|
||||
$hash = $null
|
||||
} else {
|
||||
$hash = New-MartiHash -Algorithm "SHA256" -FilePath $item.FullName
|
||||
$hash = New-MartiHash -Algorithm $Configuration.hashAlgorithm -FilePath $item.FullName
|
||||
}
|
||||
|
||||
$lattribute = Set-MartiResourceAttributes -Path $item.FullName -FileType $item.Extension.Substring(1) -ExtendedAttributes:$ExtendAttributes
|
||||
$expires = (Get-Date).AddYears(7)
|
||||
$expires = Set-DefaultExpiryDate -DocumentDate $item.LastWriteTime -Configuration $Configuration
|
||||
$version = $Configuration.version
|
||||
|
||||
$oResource = [PSCustomObject]@{
|
||||
title = $item.Name.Replace($item.Extension, "")
|
||||
title = Set-DefaultTitle -Document $item.Name -Configuration $Configuration
|
||||
uid = (New-Guid).ToString()
|
||||
documentName = $item.Name
|
||||
issuedDate = Get-Date -f "yyyy-MM-ddTHH:mm:ss"
|
||||
modified = $item.LastWriteTime.ToString("yyyy-MM-ddTHH:mm:ss")
|
||||
expires = $expires.ToString("yyyy-MM-ddTHH:mm:ss")
|
||||
state = "active"
|
||||
author = $null
|
||||
issuedDate = Get-Date -f $Configuration.dateTimeFormat
|
||||
modified = $item.LastWriteTime.ToString($Configuration.dateTimeFormat)
|
||||
expires = $expires.ToString($Configuration.dateTimeFormat)
|
||||
state = $Configuration.state
|
||||
author = $Configuration.author
|
||||
length = $item.Length
|
||||
hash = $hash
|
||||
|
||||
|
|
@ -182,7 +187,7 @@ Param(
|
|||
Write-Log "Parameter: ExcludeHash Value: $ExcludeHash "
|
||||
Write-Log ""
|
||||
|
||||
$oMarti = New-MartiDefinition
|
||||
$oMarti, $oConfig = New-MartiDefinition
|
||||
$lresource = $oMarti.resources
|
||||
|
||||
$SourceFullName = (Get-Item -Path $SourceFolder).FullName
|
||||
|
|
@ -497,6 +502,21 @@ function Set-MartiResourceAttributes {
|
|||
}
|
||||
}
|
||||
|
||||
if ($FileType -eq "MD") {
|
||||
if ($ExtendedAttributes) {
|
||||
[System.Collections.ArrayList]$lattribute = @()
|
||||
$rowCount = (Get-Content $Path).Length
|
||||
$oAttribute = [PSCustomObject]@{
|
||||
category = "dataset"
|
||||
name = "records"
|
||||
function = "count"
|
||||
comparison = "EQ"
|
||||
value = $rowCount
|
||||
}
|
||||
$lattribute += $oAttribute
|
||||
}
|
||||
}
|
||||
|
||||
if ($FileType -eq "JSON") {
|
||||
$lattribute = New-DefaultJsonAttributes
|
||||
}
|
||||
|
|
@ -678,4 +698,45 @@ function Compare-MartiResource {
|
|||
return $oResult
|
||||
}
|
||||
|
||||
function Set-DefaultExpiryDate{
|
||||
Param(
|
||||
[Parameter(Mandatory)][Datetime] $DocumentDate,
|
||||
[Parameter(Mandatory)][PSCustomObject] $Configuration
|
||||
)
|
||||
|
||||
|
||||
if ($null -eq $oConfig.expires -or $oConfig.expires -eq "") {
|
||||
$expires = $DocumentDate.AddYears(10)
|
||||
} else {
|
||||
$factors = $oConfig.expires.Split(":")
|
||||
if ($factors[0] -ne "m") {
|
||||
$expires = $DocumentDate.AddYears(10)
|
||||
} else {
|
||||
$expires = $DocumentDate.AddYears($factors[1])
|
||||
$expires = $expires.AddMonths($factors[2])
|
||||
$expires = $expires.AddDays($factors[3])
|
||||
}
|
||||
}
|
||||
|
||||
return $expires
|
||||
|
||||
}
|
||||
|
||||
|
||||
function Set-DefaultTitle{
|
||||
Param(
|
||||
[Parameter(Mandatory)][String] $Document,
|
||||
[Parameter(Mandatory)][PSCustomObject] $Configuration
|
||||
)
|
||||
|
||||
if ($null -eq $oConfig.title -or $oConfig.title -eq "") {
|
||||
$title = $Document.Replace($item.Extension, "")
|
||||
} else {
|
||||
$title = $oConfig.title.Replace("{{documentName}}", $Document.Replace($item.Extension, ""))
|
||||
$title = $title.Replace("{{documentName.ext}}", $Document)
|
||||
}
|
||||
|
||||
return $title
|
||||
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -3,7 +3,7 @@
|
|||
$script:LogPathName = ""
|
||||
|
||||
function Get-SoftwareName {
|
||||
return [String] "MARTIREFERENCE"
|
||||
return [String] "MARTILQREFERENCE"
|
||||
}
|
||||
|
||||
|
||||
|
|
|
|||
|
|
@ -21,7 +21,7 @@ from mresource import mResource
|
|||
class martiLQ:
|
||||
|
||||
_SoftwareVersion = "0.0.1"
|
||||
_default_metaFile = "##marti##.json"
|
||||
_default_metaFile = "##martilq##.json"
|
||||
|
||||
_oSoftware = {
|
||||
"extension": "software",
|
||||
|
|
|
|||
|
|
@ -20,7 +20,7 @@ from mlogging import mLogging
|
|||
class mConfiguration:
|
||||
|
||||
_SoftwareVersion = "0.0.1"
|
||||
_default_metaFile = "##marti##.json"
|
||||
_default_metaFile = "##martilq##.json"
|
||||
|
||||
_oSoftware = {
|
||||
"extension": "software",
|
||||
|
|
@ -48,7 +48,7 @@ class mConfiguration:
|
|||
"dateFormat": "2006-01-02",
|
||||
"dateTimeFormat": "2006-01-02T15:04:05+0100",
|
||||
"dataPath": "",
|
||||
"temPath": "",
|
||||
"tempPath": "",
|
||||
|
||||
"tags": None,
|
||||
"publisher": "",
|
||||
|
|
@ -63,7 +63,7 @@ class mConfiguration:
|
|||
"author": "",
|
||||
"title": "{{documentName}}",
|
||||
"state": "active",
|
||||
"expires": "m:7:0:0",
|
||||
"expires": "m:10:0:0",
|
||||
"version": "1.0",
|
||||
"urlPrefix": "",
|
||||
"encoding": "",
|
||||
|
|
|
|||
|
|
@ -0,0 +1 @@
|
|||
Generated by martiLQ Samples
|
||||
|
|
@ -1,11 +1,12 @@
|
|||
|
||||
|
||||
To execute the PowerShell scripts, please invoke from the root Marti directory and not from
|
||||
with the current directory set to ``.\test\powershel``
|
||||
with the current directory set to ``.\test\powershell``
|
||||
|
||||
``powershell
|
||||
|
||||
# To seed the test data
|
||||
# Add code for FTP fetch for BSB
|
||||
.\test\powershell\test_retrievedata.ps1
|
||||
|
||||
|
||||
|
|
|
|||
|
|
@ -8,10 +8,10 @@ try {
|
|||
$oMarti.description = "Sample execution"
|
||||
|
||||
$x = ConvertTo-Json -InputObject $oMarti -Depth 6
|
||||
Set-Content -Path ".\test\powershell\results\marti_test01.json" -Value $x
|
||||
Set-Content -Path ".\test\powershell\results\martilq_test01.json" -Value $x
|
||||
|
||||
Write-Host "Test case #2"
|
||||
$ArchiveFile = ".\test\powershell\results\marti_test02.zip"
|
||||
$ArchiveFile = ".\test\powershell\results\martilq_test02.zip"
|
||||
Compress-MartiLQ -SourceFolder ".\docs" -Filter "*" -LogPath ".\test\powershell\results\Logs" -ArchiveFile $ArchiveFile
|
||||
|
||||
Write-Host "Test case #3"
|
||||
|
|
@ -24,23 +24,23 @@ try {
|
|||
$oMarti.description = "Sample execution for ckan"
|
||||
|
||||
$x = ConvertTo-Json -InputObject $oMarti
|
||||
Set-Content -Path ".\test\powershell\results\marti_test02json.json" -Value $x
|
||||
Set-Content -Path ".\test\powershell\results\martilq_test02json.json" -Value $x
|
||||
|
||||
$x = ConvertTo-Csv -InputObject $oMarti
|
||||
Set-Content -Path ".\test\powershell\results\marti_test02csv.json" -Value $x
|
||||
Set-Content -Path ".\test\powershell\results\martilq_test02csv.json" -Value $x
|
||||
|
||||
$x = ConvertTo-Xml -As String -InputObject $oMarti -Depth 6
|
||||
Set-Content -Path ".\test\powershell\results\marti_test02xml.json" -Value $x
|
||||
Set-Content -Path ".\test\powershell\results\martilq_test02xml.json" -Value $x
|
||||
|
||||
$x = ConvertTo-Html -InputObject $oMarti
|
||||
Set-Content -Path ".\test\powershell\results\marti_test02html.json" -Value $x
|
||||
Set-Content -Path ".\test\powershell\results\martilq_test02html.json" -Value $x
|
||||
|
||||
Write-Host "Test case #5"
|
||||
$oMarti = New-MartiResource -SourcePath ".\docs\eror" -LogPath ".\test\powershell\results\Logs"
|
||||
$oMarti.description = "Sample execution with error"
|
||||
|
||||
$x = ConvertTo-Json -InputObject $oMarti
|
||||
Set-Content -Path ".\test\powershell\results\marti_test03.json" -Value $x
|
||||
Set-Content -Path ".\test\powershell\results\martilq_test03.json" -Value $x
|
||||
|
||||
}
|
||||
catch {
|
||||
|
|
|
|||
|
|
@ -5,14 +5,14 @@
|
|||
. .\source\powershell\MartiLQResource.ps1
|
||||
. .\source\powershell\MartiLQUtilities.ps1
|
||||
|
||||
$outFile = ".\test\powershell\results\marti_test_asic.json"
|
||||
$outFile = ".\test\powershell\results\martilq_test_asic.json"
|
||||
$ckan = Get-Content -Path ".\docs\source\samples\json\asic_ckan_api.json" -Raw
|
||||
$oMarti = ConvertFrom-Ckan -InputObject $ckan
|
||||
$x = ConvertTo-Json -InputObject $oMarti -Depth 5
|
||||
Set-Content -Path $outFile -Value $x
|
||||
Write-Host "Wrote converted definition to: $outFile"
|
||||
|
||||
$outFile = ".\test\powershell\results\marti_test_covid.json"
|
||||
$outFile = ".\test\powershell\results\martilq_test_covid.json"
|
||||
$covid_1 = Invoke-WebRequest "https://data.nsw.gov.au/data/api/3/action/package_show?id=793ac07d-a5f4-4851-835c-3f7158c19d15"
|
||||
$oMarti = ConvertFrom-Ckan -InputObject $covid_1
|
||||
$oMarti.description = "This data has been converted from NSW CKAN data source with URL 'https://data.nsw.gov.au/data/api/3/action/package_show?id=793ac07d-a5f4-4851-835c-3f7158c19d15'"
|
||||
|
|
@ -26,7 +26,7 @@ Write-Host "Wrote converted definition to: $outFile"
|
|||
|
||||
|
||||
# cases
|
||||
$outFile = ".\test\powershell\results\marti_test_covidcases.json"
|
||||
$outFile = ".\test\powershell\results\martilq_test_covidcases.json"
|
||||
$covid19 = "https://data.nsw.gov.au/data/api/3/action/package_show?id=3dc5dc39-40b4-4ee9-8ec6-2d862a916dcf"
|
||||
$covid_2 = Invoke-WebRequest $covid19
|
||||
$oMarti = ConvertFrom-Ckan -InputObject $covid_2
|
||||
|
|
@ -41,7 +41,7 @@ Write-Host "Wrote converted definition to: $outFile"
|
|||
|
||||
|
||||
# AFSL
|
||||
$outFile = ".\test\powershell\results\marti_test_afsl.json"
|
||||
$outFile = ".\test\powershell\results\martilq_test_afsl.json"
|
||||
$afsl = "https://data.gov.au/api/3/action/package_show?id=ab7eddce-84df-4098-bc8f-500d0d9776d1"
|
||||
$afsl_2 = Invoke-WebRequest $afsl
|
||||
$oMarti = ConvertFrom-Ckan -InputObject $afsl_2
|
||||
|
|
|
|||
Loading…
Reference in New Issue