diff --git a/.gitignore b/.gitignore index 2f2bb1b..9f44444 100644 --- a/.gitignore +++ b/.gitignore @@ -12,3 +12,6 @@ buildNumber.properties test/powershell/results/ docs/samples/powershell/test/ + +docs/samples/python/test +source/python/client/__pycache__ \ No newline at end of file diff --git a/docs/samples/powershell/Invoke-BSBSample.ps1 b/docs/samples/powershell/Invoke-SampleGenerateBsb.ps1 similarity index 58% rename from docs/samples/powershell/Invoke-BSBSample.ps1 rename to docs/samples/powershell/Invoke-SampleGenerateBsb.ps1 index 4f543a2..2535a38 100644 --- a/docs/samples/powershell/Invoke-BSBSample.ps1 +++ b/docs/samples/powershell/Invoke-SampleGenerateBsb.ps1 @@ -1,11 +1,10 @@ -. C:\Users\meerkat\source\marti\source\powershell\New-Marti.ps1 -. C:\Users\meerkat\source\marti\source\powershell\New-MartiChildItem.ps1 -. C:\Users\meerkat\source\marti\source\powershell\New-MartiItem.ps1 -. C:\Users\meerkat\source\marti\source\powershell\Get-Marti.ps1 -. C:\Users\meerkat\source\marti\source\powershell\Compress-Marti.ps1 -. C:\Users\meerkat\source\marti\source\powershell\Get-MartiFileAttributes.ps1 - +. ..\..\..\source\powershell\New-Marti.ps1 +. ..\..\..\source\powershell\New-MartiChildItem.ps1 +. ..\..\..\source\powershell\New-MartiResource.ps1 +. ..\..\..\source\powershell\Get-Marti.ps1 +. ..\..\..\source\powershell\Compress-Marti.ps1 +. ..\..\..\source\powershell\Get-MartiResourceAttributes.ps1 function PullFtpFile { @@ -77,14 +76,17 @@ function ListFtpDirectory { } $remoteDirectory = "ftp://bsb.hostedftp.com/~auspaynetftp/BSB/" -# Change local directory -$localDirectory = "./test/powershell/results/data" + +# Change local directory to suit $localDirectory = "./test" +if (!(Test-Path -Path $localDirectory)) { + New-Item -Path $localDirectory +} Write-Host "First fetch the BSB files " -ForeGroundColor Green $fileList = ListFtpDirectory -Username "anonymous" -Password "anon@merebox.com" -RemoteFile $remoteDirectory -Write-Host "File list size: $($fileList.count)" +Write-Host "File list size: $($fileList.count)" -ForegroundColor Gray Write-Host "Now iterate through the remote files and build remote marti list " -ForeGroundColor Green @@ -93,14 +95,14 @@ $oMarti = New-MartiDefinition $oMarti.title = "Remote_BSB_data" $oMarti.description = "This definition covers the remote BSB data files `r downloaded from the Australian Payment Network" $oMarti.contactPoint = "meerkat@merebox.com" -$oMarti.landingPage = "https://github.com/meerkat-manor/marti/blob/main/docs/samples/asic_ckan_api.json" +$oMarti.landingPage = "https://github.com/meerkat-manor/marti/blob/draft_specifications/docs/samples/powershell/Invoke-BSBSample.ps1" $oMarti.theme = "payment" ForEach ($item in $fileList) { if ($item -ne "" -and $item.startswith("BSBDirectory")) { PullFtpFile -Username "anonymous" -Password "anon@merebox.com" -RemoteFile ($remoteDirectory + $item) -OutputPath (Join-Path -Path $localDirectory -ChildPath $item) Write-Host "Add BSB $item file to Remote marti metadata sample " -ForeGroundColor Yellow - $oResource = New-MartiItem -SourcePath (Join-Path -Path $localDirectory -ChildPath $item) -UrlPath $remoteDirectory -LogPath ".\test\Logs" -ExtendAttributes + $oResource = New-MartiResource -SourcePath (Join-Path -Path $localDirectory -ChildPath $item) -UrlPath $remoteDirectory -LogPath ".\test\Logs" -ExtendAttributes if ($item.endswith(".txt")) { Set-AttributeValueNumber -Attributes $oResource.attributes -Key "header" -Category "dataset" -Function "count" -Value 1 Set-AttributeValueNumber -Attributes $oResource.attributes -Key "footer" -Category "dataset" -Function "count" -Value 1 @@ -117,6 +119,7 @@ ForEach ($item in $fileList) { $fileJson = Join-Path -Path $localDirectory -ChildPath "MartiBSBRemote.mri.json" $oMarti | ConvertTo-Json -depth 100 | Out-File $fileJson Write-Host "Remote marti definition file is $fileJson " -ForeGroundColor Green +Write-Host "Note that file has been given JSON extension for sample purposes" -ForeGroundColor Gray Write-Host "Now iterate through the local files and build marti ZIP " -ForeGroundColor Green @@ -125,7 +128,7 @@ $oMarti = New-MartiDefinition $oMarti.title = "Zip_BSB_data" $oMarti.description = "This definition covers the ZIP BSB data files `r downloaded from the Australian Payment Network" $oMarti.contactPoint = "meerkat@merebox.com" -$oMarti.landingPage = "https://github.com/meerkat-manor/marti/blob/main/docs/samples/asic_ckan_api.json" +$oMarti.landingPage = "https://github.com/meerkat-manor/marti/blob/draft_specifications/docs/samples/powershell/Invoke-BSBSample.ps1" $oMarti.theme = "payment" $zipFileName = "BSBDirectory.zip" @@ -135,10 +138,10 @@ if (Test-Path -Path $zipFile) { } foreach($file in Get-ChildItem $localDirectory) { - if ($file.Name.startswith("BSBDirectory") -and $file.Name -ne $zipFileName) { - Write-Host "Add BSB file $file to Local marti metadata sample " -ForeGroundColor Yellow + if ($file.Name.startswith("BSBDirectory") -and !($file.Name.EndsWith(".zip")) -and !($file.Name.EndsWith(".7z")) ) { + Write-Host "Add BSB file $file to ZIP marti metadata sample " -ForeGroundColor Yellow Compress-Archive -Path $file.FullName -DestinationPath $zipFile -Update - $oResource = New-MartiItem -SourcePath $file.FullName -UrlPath $localDirectory -LogPath ".\test\Logs" -ExtendAttributes + $oResource = New-MartiResource -SourcePath $file.FullName -UrlPath $localDirectory -LogPath ".\test\Logs" -ExtendAttributes if ($file.Extension -eq ".txt") { Set-AttributeValueNumber -Attributes $oResource.attributes -Key "header" -Category "dataset" -Function "count" -Value 1 Set-AttributeValueNumber -Attributes $oResource.attributes -Key "footer" -Category "dataset" -Function "count" -Value 1 @@ -152,13 +155,14 @@ foreach($file in Get-ChildItem $localDirectory) $oMarti.resources += $oResource } } -$oResource = New-MartiItem -SourcePath $zipFile -UrlPath $localDirectory -LogPath ".\test\Logs" -ExtendAttributes +$oResource = New-MartiResource -SourcePath $zipFile -UrlPath $localDirectory -LogPath ".\test\Logs" -ExtendAttributes Set-AttributeValueString -Attributes $oResource.attributes -Key "compression" -Category "format" -Function "algorithm" -Value "WINZIP" $oMarti.resources += $oResource $fileJson = Join-Path -Path $localDirectory -ChildPath "MartiBSBZip.mri.json" $oMarti | ConvertTo-Json -depth 100 | Out-File $fileJson Write-Host "ZIP marti definition file is $fileJson " -ForeGroundColor Green +Write-Host "Note that file has been given JSON extension for sample purposes" -ForeGroundColor Gray @@ -168,11 +172,71 @@ $oMarti = New-MartiChildItem -SourceFolder $localDirectory -UrlPath "./test" -Fi $oMarti.title = "Local_BSB_data" $oMarti.description = "This definition covers the local BSB data files `r downloaded from the Australian Payment Network" $oMarti.contactPoint = "meerkat@merebox.com" -$oMarti.landingPage = "https://github.com/meerkat-manor/marti/blob/main/docs/samples/asic_ckan_api.json" +$oMarti.landingPage = "https://github.com/meerkat-manor/marti/blob/draft_specifications/docs/samples/powershell/Invoke-BSBSample.ps1" $oMarti.theme = "payment" $fileJson = Join-Path -Path $localDirectory -ChildPath "MartiBSBLocal.mri.json" $oMarti | ConvertTo-Json -depth 100 | Out-File $fileJson Write-Host "Local marti definition file is $fileJson " -ForeGroundColor Green +Write-Host "Note that file has been given JSON extension for sample purposes" -ForeGroundColor Gray + + +Write-Host "Now create an encrypted 7ZIP file with asymmetric password protection" -ForeGroundColor Green + +$oMarti = New-MartiDefinition +$oMarti.title = "7ZIP_BSB_data" +$oMarti.description = "This definition covers the 7ZIP BSB data files `r downloaded from the Australian Payment Network" +$oMarti.contactPoint = "meerkat@merebox.com" +$oMarti.landingPage = "https://github.com/meerkat-manor/marti/blob/draft_specifications/docs/samples/powershell/Invoke-BSBSample.ps1" +$oMarti.theme = "payment" + +$zipFileName = "BSBDirectorySecure.7z" +$zipFile = Join-Path -Path $localDirectory -ChildPath $zipFileName +if (Test-Path -Path $zipFile) { + Remove-Item -Path $zipFile +} +foreach($file in Get-ChildItem $localDirectory) +{ + if ($file.Name.startswith("BSBDirectory") -and !($file.Name.EndsWith(".zip")) -and !($file.Name.EndsWith(".7z")) ) { + Write-Host "Add BSB file $file to 7ZIP marti metadata sample " -ForeGroundColor Yellow + if (Test-Path -Path $zipFile) { + Compress-7Zip -Path $file.FullName -ArchiveFileName $zipFile -Format SevenZip -Append + } else { + Compress-7Zip -Path $file.FullName -ArchiveFileName $zipFile -Format SevenZip + } + $oResource = New-MartiResource -SourcePath $file.FullName -UrlPath $localDirectory -LogPath ".\test\Logs" -ExtendAttributes + if ($file.Extension -eq ".txt") { + Set-AttributeValueNumber -Attributes $oResource.attributes -Key "header" -Category "dataset" -Function "count" -Value 1 + Set-AttributeValueNumber -Attributes $oResource.attributes -Key "footer" -Category "dataset" -Function "count" -Value 1 + Set-AttributeValueNumber -Attributes $oResource.attributes -Key "columns" -Category "dataset" -Function "count" -Value 8 + } + if ($file.Extension -eq ".csv") { + Set-AttributeValueNumber -Attributes $oResource.attributes -Key "header" -Category "dataset" -Function "count" -Value 0 + Set-AttributeValueNumber -Attributes $oResource.attributes -Key "footer" -Category "dataset" -Function "count" -Value 0 + } + $oResource.url = "@"+$zipFileName + "/" + $file.Name + $oMarti.resources += $oResource + } +} + +$noticeFile = Join-Path -Path $localDirectory -ChildPath "README.txt" +Set-Content -Path $noticeFile -Value "Generated by marti Samples" +$oResource = New-MartiResource -SourcePath $noticeFile -UrlPath $localDirectory -LogPath ".\test\Logs" +$oMarti.resources += $oResource + +$secret = "change_me_to_secure" +Compress-7Zip -Path $noticeFile -ArchiveFileName $zipFile -Append -Password $secret -EncryptFilenames + +$oResource = New-MartiResource -SourcePath $zipFile -UrlPath $localDirectory -LogPath ".\test\Logs" -ExtendAttributes +$oResource.compression = "7ZIP" +$oResource.encryption = New-Encryption -Algorithm "Passphrase" -Value $secret +Set-AttributeValueString -Attributes $oResource.attributes -Key "compression" -Category "format" -Function "algorithm" -Value "7ZIP" +$oMarti.resources += $oResource + +$fileJson = Join-Path -Path $localDirectory -ChildPath "MartiBSBSecure.mri.json" +$oMarti | ConvertTo-Json -depth 100 | Out-File $fileJson +Write-Host "Secure 7ZIP marti definition file is $fileJson " -ForeGroundColor Green +Write-Host "Note that file has been given JSON extension for sample purposes" -ForeGroundColor Gray + Write-Host "Sample execution completed" -ForeGroundColor Green diff --git a/docs/samples/powershell/Invoke-SampleGenerateBsbSecure.ps1 b/docs/samples/powershell/Invoke-SampleGenerateBsbSecure.ps1 new file mode 100644 index 0000000..88f91a8 --- /dev/null +++ b/docs/samples/powershell/Invoke-SampleGenerateBsbSecure.ps1 @@ -0,0 +1,175 @@ + +. ..\..\..\source\powershell\New-Marti.ps1 +. ..\..\..\source\powershell\New-MartiChildItem.ps1 +. ..\..\..\source\powershell\New-MartiResource.ps1 +. ..\..\..\source\powershell\Get-Marti.ps1 +. ..\..\..\source\powershell\Compress-Marti.ps1 +. ..\..\..\source\powershell\Get-MartiResourceAttributes.ps1 + + +function PullFtpFile { + param ( + [String] $RemoteFile, + [String] $OutputPath, + [String] $Username, + [String] $Password, + [int] $Buffersize = 1024 + ) + + + + $FTPRequest = [System.Net.FtpWebRequest]::Create($RemoteFile) + $FTPRequest.Credentials = New-Object System.Net.NetworkCredential($Username,$Password) + $FTPRequest.Method = [System.Net.WebRequestMethods+Ftp]::DownloadFile + $FTPRequest.UseBinary = $true + $FTPRequest.KeepAlive = $false + + $FTPResponse = $FTPRequest.GetResponse() + $ResponseStream = $FTPResponse.GetResponseStream() + + $LocalFileStream = New-Object IO.FileStream ($OutputPath,[IO.FileMode]::Create) + if ($null -eq $LocalFileStream) { + Write-Host "Write failed to file $OutputPath" + return + } + [byte[]]$ReadBuffer = New-Object byte[] $Buffersize + + # Loop through the download + do { + $ReadLength = $ResponseStream.Read($ReadBuffer,0,$Buffersize) + $LocalFileStream.Write($ReadBuffer,0,$ReadLength) + } + while ($ReadLength -gt 0) + $LocalFileStream.close() +} + + +function ListFtpDirectory { + param ( + [String] $RemoteFile, + [String] $Username, + [String] $Password, + [int] $Buffersize = 1024 + ) + + $FTPRequest = [System.Net.FtpWebRequest]::Create($RemoteFile) + $FTPRequest.Credentials = New-Object System.Net.NetworkCredential($Username,$Password) + $FTPRequest.Method = [System.Net.WebRequestMethods+Ftp]::ListDirectory + $FTPRequest.UseBinary = $false + $FTPRequest.KeepAlive = $false + + $FTPResponse = $FTPRequest.GetResponse() + $ResponseStream = $FTPResponse.GetResponseStream() + + $Encoding = new-object System.Text.AsciiEncoding + $ListBuffer = "" + [byte[]]$ReadBuffer = New-Object byte[] $Buffersize + + do { + $ReadLength = $ResponseStream.Read($ReadBuffer,0,$Buffersize) + $ListBuffer += ($Encoding.GetString($ReadBuffer, 0, $ReadLength)) + } + while ($ReadLength -ne 0) + + $list = $ListBuffer.Split([Environment]::NewLine) + return $list +} + +function New-RandomPassword { + param( + [int] $length = 30, + [String] $characters = "abcdefghiklmnoprstuvwxyzABCDEFGHKLMNOPRSTUVWXYZ1234567890!()?}][{@#*+-", + [switch] $ConvertToSecureString + ) + $password = 1..$length | ForEach-Object { Get-Random -Maximum $characters.length } + + $private:ofs="" + if ($ConvertToSecureString.IsPresent) { + return ConvertTo-SecureString -String [String]$characters[$password] -AsPlainText -Force + } else { + return [String]$characters[$password] + } +} + + + + +Write-Host "Please execute in the same directory as script" -ForeGroundColor Yellow + +$recipientKey = "CN=PeterDocs" +$remoteDirectory = "ftp://bsb.hostedftp.com/~auspaynetftp/BSB/" + +# Change local directory to suit +$localDirectory = "./test" +if (!(Test-Path -Path $localDirectory)) { + New-Item -Path $localDirectory +} + +Write-Host "First fetch the BSB files " -ForeGroundColor Green + +$fileList = ListFtpDirectory -Username "anonymous" -Password "anon@merebox.com" -RemoteFile $remoteDirectory +Write-Host "File list size: $($fileList.count)" -ForegroundColor Gray + +Write-Host "Now create an encrypted 7ZIP file with password " -ForeGroundColor Green + +$oMarti = New-MartiDefinition +$oMarti.title = "7ZIP_BSB_data" +$oMarti.description = "This definition covers the 7ZIP BSB data files `r downloaded from the Australian Payment Network" +$oMarti.contactPoint = "meerkat@merebox.com" +$oMarti.landingPage = "https://github.com/meerkat-manor/marti/blob/draft_specifications/docs/samples/powershell/Invoke-BSBSample.ps1" +$oMarti.theme = "payment" + +$zipFileName = "BSBDirectoryPKI.7z" +$zipFile = Join-Path -Path $localDirectory -ChildPath $zipFileName +if (Test-Path -Path $zipFile) { + Remove-Item -Path $zipFile +} +foreach($file in Get-ChildItem $localDirectory) +{ + if ($file.Name.startswith("BSBDirectory") -and $file.Name.EndsWith(".csv") ) { + Write-Host "Add BSB file $file to 7ZIP marti metadata sample " -ForeGroundColor Yellow + if (Test-Path -Path $zipFile) { + Compress-7Zip -Path $file.FullName -ArchiveFileName $zipFile -Format SevenZip -Append + } else { + Compress-7Zip -Path $file.FullName -ArchiveFileName $zipFile -Format SevenZip + } + $oResource = New-MartiResource -SourcePath $file.FullName -UrlPath $localDirectory -LogPath ".\test\Logs" -ExtendAttributes + if ($file.Extension -eq ".txt") { + Set-AttributeValueNumber -Attributes $oResource.attributes -Key "header" -Category "dataset" -Function "count" -Value 1 + Set-AttributeValueNumber -Attributes $oResource.attributes -Key "footer" -Category "dataset" -Function "count" -Value 1 + Set-AttributeValueNumber -Attributes $oResource.attributes -Key "columns" -Category "dataset" -Function "count" -Value 8 + } + if ($file.Extension -eq ".csv") { + Set-AttributeValueNumber -Attributes $oResource.attributes -Key "header" -Category "dataset" -Function "count" -Value 0 + Set-AttributeValueNumber -Attributes $oResource.attributes -Key "footer" -Category "dataset" -Function "count" -Value 0 + } + $oResource.url = "@"+$zipFileName + "/" + $file.Name + $oMarti.resources += $oResource + } +} + +$noticeFile = Join-Path -Path $localDirectory -ChildPath "README.txt" +Set-Content -Path $noticeFile -Value "Generated by marti Samples" +$oResource = New-MartiResource -SourcePath $noticeFile -UrlPath $localDirectory -LogPath ".\test\Logs" +$oMarti.resources += $oResource + +$secret = New-RandomPassword -Length 80 +$encryptedSecret = Protect-CmsMessage -To $recipientKey -Content $secret + +Compress-7Zip -Path $noticeFile -ArchiveFileName $zipFile -Append -Password $secret -EncryptFilenames + +$oResource = New-MartiResource -SourcePath $zipFile -UrlPath $localDirectory -LogPath ".\test\Logs" -ExtendAttributes +$oResource.compression = "7ZIP" +$oResource.encryption = New-Encryption -Algorithm "PKI" -Value $($encryptedSecret) + +Write-Debug "Secret: $secret" + +Set-AttributeValueString -Attributes $oResource.attributes -Key "compression" -Category "format" -Function "algorithm" -Value "7ZIP" +$oMarti.resources += $oResource + +$fileJson = Join-Path -Path $localDirectory -ChildPath "MartiBSB7ZipPKI.mri.json" +$oMarti | ConvertTo-Json -depth 100 | Out-File $fileJson +Write-Host "7ZIP marti definition file is $fileJson " -ForeGroundColor Green +Write-Host "Note that file has been given JSON extension for sample purposes" -ForeGroundColor Gray + +Write-Host "Sample secure execution completed" -ForeGroundColor Green diff --git a/docs/samples/python/SampleGenerateBsb.py b/docs/samples/python/SampleGenerateBsb.py new file mode 100644 index 0000000..48e2014 --- /dev/null +++ b/docs/samples/python/SampleGenerateBsb.py @@ -0,0 +1,75 @@ + + +import ftplib +import os +import json +import sys + +sys.path.insert(0, "../../../source/python/client") +from marti import * + +def ftpList(host, path): + + files = [] + with ftplib.FTP(host) as ftp: + try: + ftp.login() + ftp.cwd(path) + files = ftp.nlst() + except ftplib.all_errors as e: + print('FTP error:', e) + + return files + + +def ftpPull(host, file_remote, file_local): + + with ftplib.FTP(host) as ftp: + + try: + ftp.login() + ftp.sendcmd('TYPE I') + + with open(file_local, 'w') as fl: + res = ftp.retrlines('RETR ' + file_remote, fl.write) + if not res.startswith('226 Transfer complete'): + print('Download failed') + if os.path.isfile(file_local): + os.remove(file_local) + + except ftplib.all_errors as e: + print('FTP error:', e) + if os.path.isfile(file_local): + os.remove(file_local) + + +remote_host = 'bsb.hostedftp.com' +remote_dir = '/~auspaynetftp/BSB/' + +print("Fetch sample file list") +files = ftpList(remote_host, remote_dir) + +print("Fetch sample files") +for file_name in files: + if file_name.startswith('BSBDirectory'): + if file_name.endswith('.csv') | file_name.endswith('.txt'): + file_remote = remote_dir + file_name + file_local = './test/' + file_name + ftpPull(remote_host, file_remote, file_local) + +print("Creating marti definition") +oMarti = NewMartiDefinition() + +for file_name in files: + if file_name.startswith('BSBDirectory'): + if file_name.endswith('.csv') | file_name.endswith('.txt'): + oResource = NewMartiResource(os.path.join("./test/", file_name), "", False, True, "./test/logs") + oMarti["resources"].append(oResource) + +print("Save marti definition") +jd = json.dumps(oMarti, indent=5) + +jsonFile = open("./test/BSBDirectoryPlain.mri.json", "w") +jsonFile.write(jd) +jsonFile.close() +print("Sample completed") diff --git a/source/powershell/Get-MartiFileAttributes.ps1 b/source/powershell/Get-MartiResourceAttributes.ps1 similarity index 99% rename from source/powershell/Get-MartiFileAttributes.ps1 rename to source/powershell/Get-MartiResourceAttributes.ps1 index 6ae619f..cfa6e7f 100644 --- a/source/powershell/Get-MartiFileAttributes.ps1 +++ b/source/powershell/Get-MartiResourceAttributes.ps1 @@ -228,7 +228,7 @@ function Set-AttributeValueNumber { } -function Set-MartiFileAttributes { +function Set-MartiResourceAttributes { Param ( # File path [Parameter(Mandatory)] [String] $Path, diff --git a/source/powershell/New-MartiChildItem.ps1 b/source/powershell/New-MartiChildItem.ps1 index 4ed4a95..f82fe83 100644 --- a/source/powershell/New-MartiChildItem.ps1 +++ b/source/powershell/New-MartiChildItem.ps1 @@ -30,7 +30,7 @@ Param( Get-ChildItem $SourceFolder -Filter $Filter -Recurse:$Recurse -Force| Where-Object {!$_.PSIsContainer} | ForEach-Object { - $oResource = New-MartiItem -SourcePath $_.FullName -UrlPath $remoteDirectory -LogPath $LogPath -ExtendAttributes:$ExtendAttributes -ExcludeHash:$ExcludeHash + $oResource = New-MartiResource -SourcePath $_.FullName -UrlPath $remoteDirectory -LogPath $LogPath -ExtendAttributes:$ExtendAttributes -ExcludeHash:$ExcludeHash if ($null -ne $UrlPath -and $UrlPath -ne "") { $postfixName = $_.FullName.Replace($SourceFullName, "") diff --git a/source/powershell/New-MartiItem.ps1 b/source/powershell/New-MartiItem.ps1 index 91d4ed4..35ceffc 100644 --- a/source/powershell/New-MartiItem.ps1 +++ b/source/powershell/New-MartiItem.ps1 @@ -1,6 +1,5 @@ -function New-MartiItem -{ +function New-MartiResource { Param( [Parameter(Mandatory)][String] $SourcePath, [String] $UrlPath = "", @@ -9,12 +8,14 @@ Param( [String] $LogPath ) + $Global:MartiErrorId = "" $script:LogPathName = $LogPath Write-Debug "Parameter: LogPath Value: $LogPath " Open-Log - Write-Log "Function 'Add-MartiItem' parameters follow" + Write-Log "Function 'New-MartiResource' parameters follow" + Write-Log "Parameter: UrlPath Value: $UrlPath " Write-Log "Parameter: SourcePath Value: $SourcePath " Write-Log "Parameter: ExcludeHash Value: $ExcludeHash " Write-Log "" @@ -27,18 +28,12 @@ Param( Write-Log "Define file $($item.FullName) " if ($ExcludeHash) { - $hashAlgo = "" - } - else { - $hashAlgo = "SHA256" - } - if ($ExcludeHash) { - $hash = "" + $hash = $null } else { - $hash = (Get-FileHash -Path $item.FullName -Algorithm $hashAlgo).Hash + $hash = New-MartiHash -Algorithm "SHA256" -FilePath $item.FullName } - $lattribute = Set-MartiFileAttributes -Path $item.FullName -FileType $item.Extension.Substring(1) -ExtendedAttributes:$ExtendAttributes + $lattribute = Set-MartiResourceAttributes -Path $item.FullName -FileType $item.Extension.Substring(1) -ExtendedAttributes:$ExtendAttributes $oResource = [PSCustomObject]@{ title = $item.Name.Replace($item.Extension, "") @@ -50,14 +45,13 @@ Param( author = "" length = $item.Length hash = $hash - hashAlgo = $hashAlgo description = "" url = "" version = $version format = $item.Extension.Substring(1) - compression = "" - encryption = "" + compression = $null + encryption = $null attributes = $lattribute } @@ -83,3 +77,36 @@ Param( } +function New-MartiHash{ + Param( + [Parameter(Mandatory)][String] $Algorithm, + [String] $FilePath + [String] $Value = "" + ) + + if ($Value -eq "" -and $FilePath -ne "") { + $Value = (Get-FileHash -Path $FilePath -Algorithm $Algorithm).Hash + } + + $oHash = [PSCustomObject]@{ + algo = $Algorithm + value = $Value + } + + return $oHash +} + +function New-Encryption{ +Param( + [Parameter(Mandatory)][String] $Algorithm, + [String] $Value + +) + + $oEncryption = [PSCustomObject]@{ + algo = $Algorithm + value = $Value + } + + return $oEncryption +} diff --git a/source/python/client/marti.py b/source/python/client/marti.py new file mode 100644 index 0000000..531d08a --- /dev/null +++ b/source/python/client/marti.py @@ -0,0 +1,525 @@ + + +import ftplib +from genericpath import getsize +import os +import uuid +import json +import datetime +import getpass +import hashlib + +global_LogPathName = "" +global_SoftwareVersion = "0.0.1" +global_default_metaFile = "##marti##.mri" + +global_MartiErrorId = "" + + +def GetSoftwareName(): + return "MARTIREFERENCE" + + +def GetLogName(): + + today = datetime.datetime.today() + dateToday = today.strftime("%Y-%m-%d") + + if None == global_LogPathName or global_LogPathName == "": + return None + + if not os.path.exists(global_LogPathName): + os.mkdir(global_LogPathName) + + logName = GetSoftwareName() + "_" + dateToday + ".log" + + return os.path.join(global_LogPathName, logName) + + +def WriteLog(LogEntry): + + sFullPath = GetLogName() + + today = datetime.datetime.today() + dateToday = today.strftime("%Y-%m-%d") + + if None != sFullPath and sFullPath != "": + + if not os.path.exists(sFullPath): + print("Log path: $sFullPath") + filec = open(sFullPath, "a") + else: + filec = open(sFullPath, "a") + + filec.write(dateToday) + filec.write(".") + filec.write(LogEntry) + filec.write("\n") + filec.close() + + +def OpenLog(): + + today = datetime.datetime.today() + dateToday = today.strftime("%Y-%m-%d") + WriteLog("***********************************************************************************") + WriteLog("* Start of processing: {}".format(dateToday)) + WriteLog("***********************************************************************************") + + +def CloseLog(): + + today = datetime.datetime.today() + dateToday = today.strftime("%Y-%m-%d") + WriteLog("***********************************************************************************") + WriteLog("* End of processing: {}".format(dateToday)) + WriteLog("***********************************************************************************") + + +def NewMartiDefinition(): + + today = datetime.datetime.today() + dateToday = today.strftime("%Y-%m-%d") + + oSoftware = { + "extension": "software", + "softwareName": GetSoftwareName(), + "author": "Meerkat@merebox.com", + "version": global_SoftwareVersion + } + + publisher = getpass.getuser() + + lcustom = [] + lcustom.append(oSoftware) + + lresource = [] + + oMarti = { + "title": "", + "uid": str(uuid.uuid4()), + "resources": lresource, + + "description": "", + "modified": dateToday, + "tags": ["document", "marti"], + "publisher": publisher, + "contactPoint": "", + "accessLevel": "Confidential", + "rights": "Restricted", + "license": "", + "state": "active", + + "describedBy": "", + "landingPage": "", + "theme": "", + + "custom": lcustom + } + + return oMarti + + + +def NewMartiResource(SourcePath, UrlPath, ExcludeHash, ExtendAttributes, LogPath): + + today = datetime.datetime.today() + dateToday = today.strftime("%Y-%m-%d") + + global global_MartiErrorId + global_MartiErrorId = "" + global global_LogPathName + global_LogPathName = LogPath + + OpenLog() + WriteLog("Function 'NewMartiResource' parameters follow") + WriteLog("Parameter: SourcePath Value: {}".format(SourcePath)) + WriteLog("Parameter: UrlPath Value: {}".format(UrlPath)) + WriteLog("Parameter: ExcludeHash Value: {}".format(ExcludeHash)) + WriteLog("") + + if os.path.exists(SourcePath): + + item = os.path.basename(SourcePath) + + WriteLog("Define file {}".format(SourcePath)) + + try: + mtime = os.path.getmtime(SourcePath) + except OSError: + mtime = 0 + last_modified_date = datetime.datetime.fromtimestamp(mtime).strftime("%Y-%m-%d %H:%M:%S") + + if ExcludeHash: + hash = None + else: + hash = NewMartiHash(Algorithm="SHA256", FilePath=SourcePath, Value="") + + lattribute = SetMartiResourceAttributes(SourcePath, os.path.splitext(SourcePath)[1][1:], ExtendAttributes) + + oResource = { + "title": item.replace(os.path.splitext(SourcePath)[1], ""), + "uid": str(uuid.uuid4()), + "documentName": item, + "issuedDate": dateToday, + "modified": last_modified_date, + "state": "active", + "author": "", + "length": os.path.getsize(SourcePath), + "hash": hash, + + "description": "", + "url": "", + "version": global_SoftwareVersion, + "format": os.path.splitext(SourcePath)[1][1:], + "compression": None, + "encryption": None, + + "attributes": lattribute + } + + if None != UrlPath and UrlPath != "": + if UrlPath[UrlPath.Length-1] == "/" or UrlPath[UrlPath.Length-1] == "\\": + oResource["url"] = UrlPath.replace("\\", "/") + item + else: + oResource["url"] = UrlPath.replace("\\", "/") + "/" + item + + + else: + global_MartiErrorId = "MRI2001" + message = "Document '{}' not found or is a folder".format(SourcePath) + WriteLog(message + " " + global_MartiErrorId) + CloseLog() + raise Exception(message) + + CloseLog() + + return oResource + + + +def NewMartiHash(Algorithm, FilePath, Value=""): + + if Value == "" and FilePath != "": + if Algorithm == "SHA256": + sha_hash = hashlib.sha256() + with open(FilePath,"rb") as fh: + for byte_block in iter(lambda: fh.read(4096),b""): + sha_hash.update(byte_block) + Value = sha_hash.hexdigest() + if Algorithm == "SHA512": + sha_hash = hashlib.sha512() + with open(FilePath,"rb") as fh: + for byte_block in iter(lambda: fh.read(4096),b""): + sha_hash.update(byte_block) + Value = sha_hash.hexdigest() + if Algorithm == "MD5": + sha_hash = hashlib.md5() + with open(FilePath,"rb") as fh: + for byte_block in iter(lambda: fh.read(4096),b""): + sha_hash.update(byte_block) + Value = sha_hash.hexdigest() + + oHash = { + "algo": Algorithm, + "value": Value + } + + return oHash + + +def NewEncryption(Algorithm, Value): + + oEncryption = { + "algo": Algorithm, + "value": Value + } + + return oEncryption + + + + + + +def SetMartiAttribute(Attributes, ACategory, AName, AFunction, Comparison, Value): + + matched = False + + for attr in Attributes: + + if attr["category"] == ACategory and attr["name"] == AName and attr["function"] == AFunction: + matched = True + attr["comparison"] = Comparison + attr["value"] = Value + + + if not matched: + + oAttribute = { + "category": ACategory, + "name": AName, + "function": AFunction, + "comparison": Comparison, + "value": Value + } + + Attributes.append(oAttribute) + + + return Attributes + + + +def NewDefaultCsvAttributes(): + + lattribute = [] + + oAttribute = { + "category": "dataset", + "name": "header", + "function": "count", + "comparison": "NA", + "value": 1 + } + lattribute.append(oAttribute) + + oAttribute = { + "category": "dataset", + "name": "footer", + "function": "count", + "comparison": "NA", + "value": 0 + } + lattribute.append(oAttribute) + + oAttribute = { + "category": "format", + "name": "separator", + "function": "value", + "comparison": "NA", + "value": "," + } + lattribute.append(oAttribute) + + oAttribute = { + "category": "format", + "name": "columns", + "function": "value", + "comparison": "NA", + "value": "," + } + lattribute.append(oAttribute) + + oAttribute = { + "category": "dataset", + "name": "records", + "function": "count", + "comparison": "NA", + "value": 0 + } + lattribute.append(oAttribute) + + oAttribute = { + "category": "dataset", + "name": "columns", + "function": "count", + "comparison": "NA", + "value": 0 + } + lattribute.append(oAttribute) + + return lattribute + + + +def NewDefaultJsonAttributes(): + + lattribute = [] + + oAttribute = { + "category": "format", + "name": "list", + "function": "offset", + "comparison": "NA", + "value": "," + } + lattribute.append(oAttribute) + + oAttribute = { + "category": "format", + "name": "columns", + "function": "value", + "comparison": "NA", + "value": "," + } + lattribute.append(oAttribute) + + oAttribute = { + "category": "dataset", + "name": "records", + "function": "count", + "comparison": "NA", + "value": 0 + } + lattribute.append(oAttribute) + + oAttribute = { + "category": "dataset", + "name": "columns", + "function": "count", + "comparison": "NA", + "value": 0 + } + lattribute.append(oAttribute) + + return lattribute + + +def NewDefaultZipAttributes(CompressionType = "ZIP",Encryption = ""): + + lattribute = [] + + oAttribute = { + "category": "format", + "name": "compression", + "function": "algorithm", + "comparison": "NA", + "value": CompressionType + } + lattribute.append(oAttribute) + + oAttribute = { + "category": "format", + "name": "encryption", + "function": "algorithm", + "comparison": "NA", + "value": Encryption + } + lattribute.append(oAttribute) + + oAttribute = { + "category": "dataset", + "name": "files", + "function": "count", + "comparison": "NA", + "value": 0 + } + lattribute.append(oAttribute) + + return lattribute + + +def SetAttributeValueString(Attributes,Category,Key,Function,Value,Comparison = "EQ"): + + for item in Attributes: + + if item["category"] == Category and item["name"] == Key and item["function"] == Function: + if item["comparison"] == "NA" or item["comparison"] == Comparison: + item["comparison"] = Comparison + item["value"] = Value + return + + # Add the attribute + oAttribute = { + "category": Category, + "name": Key, + "function": Function, + "comparison": Comparison, + "value": Value + } + + Attributes.append(oAttribute) + + return + + +def SetAttributeValueNumber(Attributes,Category,Key,Function,Value,Comparison = "EQ"): + + for item in Attributes: + + if item["category"] == Category and item["name"] == Key and item["function"] == Function: + if item["comparison"] == "NA" or item["comparison"] == Comparison: + item["comparison"] = Comparison + item["value"] = Value + return + + + # Add the attribute + oAttribute = { + "category": Category, + "name": Key, + "function": Function, + "comparison": Comparison, + "value": Value + } + + Attributes.append(oAttribute) + + return + + +def SetMartiResourceAttributes(Path,FileType,ExtendedAttributes): + + lattribute = None + + if FileType == "csv": + lattribute = NewDefaultCsvAttributes() + + if ExtendedAttributes: + delimiter = "," + rowCount = 0 + colCount = 0 + # csvData = Import-Csv $Path -Delimiter $delimiter + # foreach ($datum in $csvData) { + # $cc = (Get-Member -InputObject $datum -type NoteProperty).count + # if ($colCount -lt $cc) { + # $colCount = $cc + # } + # $rowCount += 1 + # } + SetAttributeValueNumber(lattribute, "records", "dataset", "count", rowCount) + SetAttributeValueNumber(lattribute, "columns", "dataset", "count", colCount) + + + if FileType == "txt": + lattribute = NewDefaultCsvAttributes() + + if ExtendedAttributes: + delimiter = "`t" + rowCount = 0 + colCount = 0 + # $csvData = Import-Csv $Path -Delimiter $delimiter + # foreach ($datum in $csvData) { + # $cc = (Get-Member -InputObject $datum -type NoteProperty).count + # if ($colCount -lt $cc) { + # $colCount = $cc + # } + # $rowCount += 1 + # } + SetAttributeValueNumber(lattribute, "records", "dataset", "count", rowCount) + SetAttributeValueNumber(lattribute, "columns", "dataset", "count", colCount) + + + if FileType == "json": + lattribute = NewDefaultJsonAttributes() + + + if FileType == "zip": + lattribute = NewDefaultZipAttributes("ZIP") + if ExtendedAttributes: + # $shell = New-Object -Com Shell.Application + # $zipFile = $shell.NameSpace($Path) + # $items = $zipFile.Items() + SetAttributeValueNumber(lattribute, "files", "dataset", "count", -1) + + if FileType == "7z": + lattribute = NewDefaultZipAttributes("7Z") + + if lattribute == None: + lattribute = [] + + + return lattribute + + diff --git a/test/powershell/test_Marti.ps1 b/test/powershell/test_Marti.ps1 index 97c1d8b..1def922 100644 --- a/test/powershell/test_Marti.ps1 +++ b/test/powershell/test_Marti.ps1 @@ -20,7 +20,7 @@ Write-Host "Get item Title: $($y.title)" Write-Host "Get item Url: $($y.url)" Write-Host "Test case #4" -$oMarti = New-MartiItem -SourcePath ".\docs\ckan.md" -LogPath ".\test\powershell\results\Logs" +$oMarti = New-MartiResource -SourcePath ".\docs\ckan.md" -LogPath ".\test\powershell\results\Logs" $oMarti.description = "Sample execution for ckan" $x = ConvertTo-Json -InputObject $oMarti @@ -36,7 +36,7 @@ $x = ConvertTo-Html -InputObject $oMarti Set-Content -Path ".\test\powershell\results\marti_test02.mri.html" -Value $x Write-Host "Test case #5" -$oMarti = New-MartiItem -SourcePath ".\docs\eror" -LogPath ".\test\powershell\results\Logs" +$oMarti = New-MartiResource -SourcePath ".\docs\eror" -LogPath ".\test\powershell\results\Logs" $oMarti.description = "Sample execution with error" $x = ConvertTo-Json -InputObject $oMarti diff --git a/test/powershell/test_MartiData2.ps1 b/test/powershell/test_MartiData2.ps1 index 8c540fa..47f649f 100644 --- a/test/powershell/test_MartiData2.ps1 +++ b/test/powershell/test_MartiData2.ps1 @@ -7,7 +7,7 @@ $bsbFile = ".\test\powershell\results\data\bsb.csv" Write-Host ">>>>>>Test case #1" -$x = New-MartiItem -SourcePath $bsbFile -UrlPath "" -ExcludeHash -LogPath ".\test\powershell\results\Logs" +$x = New-MartiResource -SourcePath $bsbFile -UrlPath "" -ExcludeHash -LogPath ".\test\powershell\results\Logs" Write-Host ">>>>>>Test case #2" $x.resources @@ -25,7 +25,7 @@ $y $covidFile = ".\test\powershell\results\data\covid-case-locations-20210920-1315.json" Write-Host ">>>>>>Test case #5" -$x = New-MartiItem -SourcePath $covidFile -UrlPath "" -ExcludeHash -LogPath ".\test\powershell\results\Logs" +$x = New-MartiResource -SourcePath $covidFile -UrlPath "" -ExcludeHash -LogPath ".\test\powershell\results\Logs" Write-Host ">>>>>>Test case #6" $x.resources diff --git a/test/powershell/test_MartiData3.ps1 b/test/powershell/test_MartiData3.ps1 index ddb7947..9e7f2e1 100644 --- a/test/powershell/test_MartiData3.ps1 +++ b/test/powershell/test_MartiData3.ps1 @@ -6,7 +6,7 @@ $covidFile = ".\test\powershell\results\data\covid-case-locations-20210920-1315.json" -$x = New-MartiItem -SourcePath $covidFile -UrlPath "" -ExcludeHash -LogPath ".\test\powershell\results\Logs" +$x = New-MartiResource -SourcePath $covidFile -UrlPath "" -ExcludeHash -LogPath ".\test\powershell\results\Logs" [System.Collections.ArrayList] $attr = Set-MartiAttribute -Attributes $x.resources[0].attributes -ACategory "dataset" -AName "records" -AFunction "count" -comparison "EQ" -value 516 $attr = Set-MartiAttribute -Attributes $attr -ACategory "dataset" -AName "columns" -AFunction "count" -comparison "EQ" -value 12