Change from format tp contentType
parent
2428dcbf58
commit
28168c5fc1
|
|
@ -27,3 +27,4 @@ source/golang/client/src/test
|
|||
source/golang/client/src/martilq_client*
|
||||
source/golang/client/src/*.exe
|
||||
|
||||
source/golang/server/main
|
||||
|
|
|
|||
|
|
@ -105,7 +105,7 @@ sample can be generated using the GOLANG client program with parameters:
|
|||
|
||||
```json
|
||||
{
|
||||
"content-type": "application/vnd.martilq.json",
|
||||
"contentType": "application/vnd.martilq.json",
|
||||
"title": "GEN001",
|
||||
"uid": "9a0a7edb-dd81-4fc5-a6cb-c5716eda7b51",
|
||||
"description": "Simple example",
|
||||
|
|
@ -140,7 +140,7 @@ sample can be generated using the GOLANG client program with parameters:
|
|||
"description": "",
|
||||
"url": "http://localhost/martilq/martilq.md",
|
||||
"version": "",
|
||||
"content-type": "",
|
||||
"contentType": "",
|
||||
"encoding": "UTF-8",
|
||||
"compression": "",
|
||||
"encryption": "",
|
||||
|
|
|
|||
|
|
@ -1,5 +1,5 @@
|
|||
{
|
||||
"content-type": "application/vnd.martilq.json",
|
||||
"contentType": "application/vnd.martilq.json",
|
||||
"title": "SampleFetchBSB",
|
||||
"uid": "08e22a4d-5a11-4b19-9172-feb36601009e",
|
||||
"description": "",
|
||||
|
|
@ -33,7 +33,7 @@
|
|||
"description": "",
|
||||
"url": "ftp://bsb.hostedftp.com/~auspaynetftp/BSB/BSBDirectoryAug21-305.csv",
|
||||
"version": "",
|
||||
"content-type": "text/csv",
|
||||
"contentType": "text/csv",
|
||||
"compression": null,
|
||||
"encryption": null,
|
||||
"attributes": [
|
||||
|
|
@ -98,7 +98,7 @@
|
|||
"description": "",
|
||||
"url": "ftp://bsb.hostedftp.com/~auspaynetftp/BSB/BSBDirectoryAug21-305.txt",
|
||||
"version": "",
|
||||
"content-type": "text/plain",
|
||||
"contentType": "text/plain",
|
||||
"compression": null,
|
||||
"encryption": null,
|
||||
"attributes": [
|
||||
|
|
@ -163,7 +163,7 @@
|
|||
"description": "",
|
||||
"url": "ftp://bsb.hostedftp.com/~auspaynetftp/BSB/BSBDirectoryJul21-304.csv",
|
||||
"version": "",
|
||||
"content-type": "text/csv",
|
||||
"contentType": "text/csv",
|
||||
"compression": null,
|
||||
"encryption": null,
|
||||
"attributes": [
|
||||
|
|
@ -228,7 +228,7 @@
|
|||
"description": "",
|
||||
"url": "ftp://bsb.hostedftp.com/~auspaynetftp/BSB/BSBDirectoryJul21-304.txt",
|
||||
"version": "",
|
||||
"content-type": "text/plain",
|
||||
"contentType": "text/plain",
|
||||
"compression": null,
|
||||
"encryption": null,
|
||||
"attributes": [
|
||||
|
|
@ -293,7 +293,7 @@
|
|||
"description": "",
|
||||
"url": "ftp://bsb.hostedftp.com/~auspaynetftp/BSB/BSBDirectorySep21-306.csv",
|
||||
"version": "",
|
||||
"content-type": "text/csv",
|
||||
"contentType": "text/csv",
|
||||
"compression": null,
|
||||
"encryption": null,
|
||||
"attributes": [
|
||||
|
|
@ -358,7 +358,7 @@
|
|||
"description": "",
|
||||
"url": "ftp://bsb.hostedftp.com/~auspaynetftp/BSB/BSBDirectorySep21-306.txt",
|
||||
"version": "",
|
||||
"content-type": "text/plain",
|
||||
"contentType": "text/plain",
|
||||
"compression": null,
|
||||
"encryption": null,
|
||||
"attributes": [
|
||||
|
|
|
|||
|
|
@ -1,5 +1,5 @@
|
|||
{
|
||||
"content-type": "application/vnd.martilq.json",
|
||||
"contentType": "application/vnd.martilq.json",
|
||||
"title": "SampleFetchBSB",
|
||||
"uid": "08e22a4d-5a11-4b19-9172-feb36601009e",
|
||||
"description": "",
|
||||
|
|
@ -33,7 +33,7 @@
|
|||
"description": "",
|
||||
"url": "http://apnedata.merebox.com.s3.ap-southeast-2.amazonaws.com/au/bsb/BSBDirectoryAug21-305.csv",
|
||||
"version": "",
|
||||
"content-type": "text/csv",
|
||||
"contentType": "text/csv",
|
||||
"compression": null,
|
||||
"encryption": null,
|
||||
"attributes": [
|
||||
|
|
@ -98,7 +98,7 @@
|
|||
"description": "",
|
||||
"url": "http://apnedata.merebox.com.s3.ap-southeast-2.amazonaws.com/au/bsb/BSBDirectoryJul21-304.csv",
|
||||
"version": "",
|
||||
"content-type": "text/csv",
|
||||
"contentType": "text/csv",
|
||||
"compression": null,
|
||||
"encryption": null,
|
||||
"attributes": [
|
||||
|
|
@ -163,7 +163,7 @@
|
|||
"description": "",
|
||||
"url": "http://apnedata.merebox.com.s3.ap-southeast-2.amazonaws.com/au/bsb/BSBDirectorySep21-306.csv",
|
||||
"version": "",
|
||||
"content-type": "text/csv",
|
||||
"contentType": "text/csv",
|
||||
"compression": null,
|
||||
"encryption": null,
|
||||
"attributes": [
|
||||
|
|
|
|||
|
|
@ -14,7 +14,7 @@ The **martiLQ** document
|
|||
|
||||
```json
|
||||
{
|
||||
"content-type": "application/vnd.martilq.json",
|
||||
"contentType": "application/vnd.martilq.json",
|
||||
"title": "GEN001",
|
||||
"uid": "cff090b2-5845-490c-bd81-88b3a9267b5a",
|
||||
"description": "Simple example with no config",
|
||||
|
|
@ -49,7 +49,7 @@ The **martiLQ** document
|
|||
"description": "",
|
||||
"url": "file://martiLQ.md",
|
||||
"version": "",
|
||||
"content-type": "",
|
||||
"contentType": "",
|
||||
"encoding": "UTF-8",
|
||||
"compression": "",
|
||||
"encryption": "",
|
||||
|
|
|
|||
|
|
@ -41,7 +41,7 @@ The **martiLQ** document
|
|||
|
||||
```json
|
||||
{
|
||||
"content-type": "application/vnd.martilq.json",
|
||||
"contentType": "application/vnd.martilq.json",
|
||||
"title": "GEN002",
|
||||
"uid": "42532ae8-4d42-4875-b7c8-bcb7a5c38eaa",
|
||||
"description": "Simple example",
|
||||
|
|
@ -79,7 +79,7 @@ The **martiLQ** document
|
|||
"description": "",
|
||||
"url": "https://github.com/meerkat-manor/marti/blob/draft_specifications/docs/source/martiLQ.md",
|
||||
"version": "",
|
||||
"content-type": "",
|
||||
"contentType": "",
|
||||
"encoding": "UTF-8",
|
||||
"compression": "",
|
||||
"encryption": "",
|
||||
|
|
|
|||
|
|
@ -16,7 +16,7 @@ The **martiLQ** document
|
|||
|
||||
```json
|
||||
{
|
||||
"content-type": "application/vnd.martilq.json",
|
||||
"contentType": "application/vnd.martilq.json",
|
||||
"title": "GEN002",
|
||||
"uid": "a4cb9d10-4d87-455b-87ec-64003147b0cd",
|
||||
"description": "Directory example",
|
||||
|
|
@ -54,7 +54,7 @@ The **martiLQ** document
|
|||
"description": "",
|
||||
"url": "https://github.com/meerkat-manor/marti/blob/draft_specifications/docs/source/README.md",
|
||||
"version": "",
|
||||
"content-type": "",
|
||||
"contentType": "",
|
||||
"encoding": "UTF-8",
|
||||
"compression": "",
|
||||
"encryption": "",
|
||||
|
|
@ -87,7 +87,7 @@ The **martiLQ** document
|
|||
"description": "",
|
||||
"url": "https://github.com/meerkat-manor/marti/blob/draft_specifications/docs/source/attributes.md",
|
||||
"version": "",
|
||||
"content-type": "",
|
||||
"contentType": "",
|
||||
"encoding": "UTF-8",
|
||||
"compression": "",
|
||||
"encryption": "",
|
||||
|
|
@ -120,7 +120,7 @@ The **martiLQ** document
|
|||
"description": "",
|
||||
"url": "https://github.com/meerkat-manor/marti/blob/draft_specifications/docs/source/ckan.md",
|
||||
"version": "",
|
||||
"content-type": "",
|
||||
"contentType": "",
|
||||
"encoding": "UTF-8",
|
||||
"compression": "",
|
||||
"encryption": "",
|
||||
|
|
@ -153,7 +153,7 @@ The **martiLQ** document
|
|||
"description": "",
|
||||
"url": "https://github.com/meerkat-manor/marti/blob/draft_specifications/docs/source/comparison.md",
|
||||
"version": "",
|
||||
"content-type": "",
|
||||
"contentType": "",
|
||||
"encoding": "UTF-8",
|
||||
"compression": "",
|
||||
"encryption": "",
|
||||
|
|
@ -186,7 +186,7 @@ The **martiLQ** document
|
|||
"description": "",
|
||||
"url": "https://github.com/meerkat-manor/marti/blob/draft_specifications/docs/source/custom.md",
|
||||
"version": "",
|
||||
"content-type": "",
|
||||
"contentType": "",
|
||||
"encoding": "UTF-8",
|
||||
"compression": "",
|
||||
"encryption": "",
|
||||
|
|
@ -219,7 +219,7 @@ The **martiLQ** document
|
|||
"description": "",
|
||||
"url": "https://github.com/meerkat-manor/marti/blob/draft_specifications/docs/source/magda.md",
|
||||
"version": "",
|
||||
"content-type": "",
|
||||
"contentType": "",
|
||||
"encoding": "UTF-8",
|
||||
"compression": "",
|
||||
"encryption": "",
|
||||
|
|
@ -252,7 +252,7 @@ The **martiLQ** document
|
|||
"description": "",
|
||||
"url": "https://github.com/meerkat-manor/marti/blob/draft_specifications/docs/source/martiLQ.md",
|
||||
"version": "",
|
||||
"content-type": "",
|
||||
"contentType": "",
|
||||
"encoding": "UTF-8",
|
||||
"compression": "",
|
||||
"encryption": "",
|
||||
|
|
@ -285,7 +285,7 @@ The **martiLQ** document
|
|||
"description": "",
|
||||
"url": "https://github.com/meerkat-manor/marti/blob/draft_specifications/docs/source/quality.md",
|
||||
"version": "",
|
||||
"content-type": "",
|
||||
"contentType": "",
|
||||
"encoding": "UTF-8",
|
||||
"compression": "",
|
||||
"encryption": "",
|
||||
|
|
@ -318,7 +318,7 @@ The **martiLQ** document
|
|||
"description": "",
|
||||
"url": "https://github.com/meerkat-manor/marti/blob/draft_specifications/docs/source/references.md",
|
||||
"version": "",
|
||||
"content-type": "",
|
||||
"contentType": "",
|
||||
"encoding": "UTF-8",
|
||||
"compression": "",
|
||||
"encryption": "",
|
||||
|
|
@ -351,7 +351,7 @@ The **martiLQ** document
|
|||
"description": "",
|
||||
"url": "https://github.com/meerkat-manor/marti/blob/draft_specifications/docs/source/resources.md",
|
||||
"version": "",
|
||||
"content-type": "",
|
||||
"contentType": "",
|
||||
"encoding": "UTF-8",
|
||||
"compression": "",
|
||||
"encryption": "",
|
||||
|
|
@ -384,7 +384,7 @@ The **martiLQ** document
|
|||
"description": "",
|
||||
"url": "https://github.com/meerkat-manor/marti/blob/draft_specifications/docs/source/what.md",
|
||||
"version": "",
|
||||
"content-type": "",
|
||||
"contentType": "",
|
||||
"encoding": "UTF-8",
|
||||
"compression": "",
|
||||
"encryption": "",
|
||||
|
|
@ -417,7 +417,7 @@ The **martiLQ** document
|
|||
"description": "",
|
||||
"url": "https://github.com/meerkat-manor/marti/blob/draft_specifications/docs/source/when.md",
|
||||
"version": "",
|
||||
"content-type": "",
|
||||
"contentType": "",
|
||||
"encoding": "UTF-8",
|
||||
"compression": "",
|
||||
"encryption": "",
|
||||
|
|
@ -450,7 +450,7 @@ The **martiLQ** document
|
|||
"description": "",
|
||||
"url": "https://github.com/meerkat-manor/marti/blob/draft_specifications/docs/source/who.md",
|
||||
"version": "",
|
||||
"content-type": "",
|
||||
"contentType": "",
|
||||
"encoding": "UTF-8",
|
||||
"compression": "",
|
||||
"encryption": "",
|
||||
|
|
@ -483,7 +483,7 @@ The **martiLQ** document
|
|||
"description": "",
|
||||
"url": "https://github.com/meerkat-manor/marti/blob/draft_specifications/docs/source/why.md",
|
||||
"version": "",
|
||||
"content-type": "",
|
||||
"contentType": "",
|
||||
"encoding": "UTF-8",
|
||||
"compression": "",
|
||||
"encryption": "",
|
||||
|
|
|
|||
|
|
@ -19,7 +19,7 @@ The **martiLQ** document
|
|||
|
||||
```json
|
||||
{
|
||||
"content-type": "application/vnd.martilq.json",
|
||||
"contentType": "application/vnd.martilq.json",
|
||||
"title": "GEN004",
|
||||
"uid": "207096a1-aac1-4b66-9748-707f07b63691",
|
||||
"description": "Directory example with filter",
|
||||
|
|
@ -54,7 +54,7 @@ The **martiLQ** document
|
|||
"description": "",
|
||||
"url": "file://README.md",
|
||||
"version": "",
|
||||
"content-type": "",
|
||||
"contentType": "",
|
||||
"encoding": "UTF-8",
|
||||
"compression": "",
|
||||
"encryption": "",
|
||||
|
|
@ -87,7 +87,7 @@ The **martiLQ** document
|
|||
"description": "",
|
||||
"url": "file://references.md",
|
||||
"version": "",
|
||||
"content-type": "",
|
||||
"contentType": "",
|
||||
"encoding": "UTF-8",
|
||||
"compression": "",
|
||||
"encryption": "",
|
||||
|
|
@ -120,7 +120,7 @@ The **martiLQ** document
|
|||
"description": "",
|
||||
"url": "file://resources.md",
|
||||
"version": "",
|
||||
"content-type": "",
|
||||
"contentType": "",
|
||||
"encoding": "UTF-8",
|
||||
"compression": "",
|
||||
"encryption": "",
|
||||
|
|
@ -153,7 +153,7 @@ The **martiLQ** document
|
|||
"description": "",
|
||||
"url": "file://samples/README.md",
|
||||
"version": "",
|
||||
"content-type": "",
|
||||
"contentType": "",
|
||||
"encoding": "UTF-8",
|
||||
"compression": "",
|
||||
"encryption": "",
|
||||
|
|
@ -186,7 +186,7 @@ The **martiLQ** document
|
|||
"description": "",
|
||||
"url": "file://samples/json/README.md",
|
||||
"version": "",
|
||||
"content-type": "",
|
||||
"contentType": "",
|
||||
"encoding": "UTF-8",
|
||||
"compression": "",
|
||||
"encryption": "",
|
||||
|
|
|
|||
|
|
@ -76,18 +76,18 @@ for file_name in files:
|
|||
oResource = mlq.NewMartiLQResource(os.path.join(test_dir, file_name), "", False, True)
|
||||
oMarti["resources"].append(oResource)
|
||||
|
||||
mlq.CloseLog()
|
||||
mlq.Close()
|
||||
|
||||
|
||||
print("Save martiLQ definition")
|
||||
jsonFile = open(os.path.join(test_dir, "BSBDirectoryPlain.json"), "w")
|
||||
jsonFile = open(os.path.join(test_dir, "martilq_bsb.json"), "w")
|
||||
jsonFile.write(json.dumps(oMarti, indent=5))
|
||||
jsonFile.close()
|
||||
print("Base sample JSON written: BSBDirectoryPlain.json")
|
||||
print("Base sample JSON written: martilq_bsb.json")
|
||||
|
||||
|
||||
print("Creating martiLQ ZIP file")
|
||||
zipFileName = "BSBDirectory.zip"
|
||||
zipFileName = "martilq_bsb.zip"
|
||||
fileZipCount = 0
|
||||
|
||||
mlq = martiLQ()
|
||||
|
|
@ -108,24 +108,26 @@ with zipfile.ZipFile(os.path.join(test_dir, zipFileName), "w", compression=zipfi
|
|||
|
||||
|
||||
oResource = mlq.NewMartiLQResource(os.path.join(test_dir, zipFileName), "", False, True)
|
||||
oResource["url"] = os.path.join(test_dir, zipFileName)
|
||||
mlq.SetAttributeValueString(Attributes=oResource["attributes"], Key="compression", Category="format", Function="algo", Value="WINZIP")
|
||||
mlq.SetAttributeValueNumber(Attributes=oResource["attributes"], Key="files", Category="dataset", Function="count", Value=fileZipCount)
|
||||
oResource["url"] = test_dir + "/" + zipFileName
|
||||
|
||||
mResource.SetAttributeValueString(oResource, Key="compression", Category="format", Function="algo", Value="WINZIP")
|
||||
mResource.SetAttributeValueNumber(oResource, Key="files", Category="dataset", Function="count", Value=fileZipCount)
|
||||
|
||||
oMarti["resources"].append(oResource)
|
||||
|
||||
mlq.CloseLog()
|
||||
mlq.Close()
|
||||
|
||||
print("Save martiLQ ZIP definition")
|
||||
jsonFile = open(os.path.join(test_dir, "MartiLQ_BSBZip.json"), "w")
|
||||
jsonFile = open(os.path.join(test_dir, "martilq_bsb_zip.json"), "w")
|
||||
jsonFile.write(json.dumps(oMarti, indent=5))
|
||||
jsonFile.close()
|
||||
print("ZIP sample JSON written: MartiLQ_BSBZip.json")
|
||||
print("ZIP sample JSON written: martilq_bsb_zip.json")
|
||||
|
||||
|
||||
|
||||
print("Sample completed: SampleGenerateBsb.py")
|
||||
print("Sample completed: SampleGenerateFtpBsb.py")
|
||||
|
||||
lqresults, testError = mlq.TestMartiDefinition(os.path.join(test_dir, "BSBDirectoryPlain.json"))
|
||||
lqresults, testError = mlq.TestMartiDefinition(os.path.join(test_dir, "martilq_bsb.json"))
|
||||
|
||||
testfile = open(os.path.join(test_dir, "LoadQualityTest01.csv"), "w+", newline ="")
|
||||
with testfile:
|
||||
|
|
|
|||
|
|
@ -26,11 +26,13 @@ A web UI exists as a functionnal demonstration to view a collection of
|
|||
|
||||
## Docker
|
||||
|
||||
Please remember to copy documents into "docs" folder.
|
||||
|
||||
```
|
||||
|
||||
go env -w GOOS=linux
|
||||
go env -w GOARCH=386
|
||||
go build ./main.go
|
||||
go build ./src/main.go
|
||||
|
||||
docker build -t martilq-go-server:latest .
|
||||
```
|
||||
|
|
|
|||
|
|
@ -22,7 +22,7 @@ import (
|
|||
|
||||
|
||||
type Marti struct {
|
||||
Content_type string `json:"content-type"`
|
||||
Content_type string `json:"contentType"`
|
||||
Title string `json:"title"`
|
||||
Uid string `json:"uid"`
|
||||
|
||||
|
|
|
|||
|
|
@ -29,8 +29,9 @@ type Resource struct {
|
|||
|
||||
Description string `json:"description"`
|
||||
Url string `json:"url"`
|
||||
Structure string `json:"structure"`
|
||||
Version string `json:"version"`
|
||||
Content_Type string `json:"content-type"`
|
||||
ContentType string `json:"contentType"`
|
||||
Encoding string `json:"encoding"`
|
||||
Compression string `json:"compression"`
|
||||
Encryption string `json:"encryption"`
|
||||
|
|
@ -112,7 +113,7 @@ func NewMartiLQResource(config configuration, sourcePath string, urlPath string,
|
|||
parts := strings.Split(sourcePath,".")
|
||||
extension := parts[len(parts)-1]
|
||||
|
||||
r.Content_Type = mime.TypeByExtension("."+extension)
|
||||
r.ContentType = mime.TypeByExtension("."+extension)
|
||||
records := 0
|
||||
columns := -1
|
||||
|
||||
|
|
|
|||
|
|
@ -1,14 +1,15 @@
|
|||
FROM busybox
|
||||
|
||||
LABEL version="202111A"
|
||||
LABEL description="matiLQ Go web UI by Meerkat Manor "
|
||||
LABEL version="202111B"
|
||||
LABEL description="martiLQ Go web UI by Meerkat Manor "
|
||||
LABEL authors="meerkat@merebox.com"
|
||||
|
||||
COPY ./static /home/static
|
||||
COPY ./main /home/main
|
||||
COPY ./docs /home/docs
|
||||
COPY ./data /data
|
||||
COPY ./src/static /martilq/static
|
||||
COPY ./src/main /martilq/main
|
||||
COPY ./src/docs /martilq/docs
|
||||
COPY ./src/template /martilq/template
|
||||
COPY ./src/data /martilq/data
|
||||
|
||||
EXPOSE 8080
|
||||
|
||||
ENTRYPOINT exec /home/main -p 8080 -s /home/static -data /data/ -docs /home/docs
|
||||
ENTRYPOINT exec /martilq/main -p 8080 -static /martilq/static -data /martilq/data/ -docs /martilq/docs -template /martilq/template
|
||||
|
|
|
|||
|
|
@ -5,11 +5,11 @@ services:
|
|||
web:
|
||||
hostname: martilq-web
|
||||
image: martilq-go-server:latest
|
||||
container_name: matilq-web
|
||||
container_name: martilq-web
|
||||
|
||||
ports:
|
||||
- 8080:8080
|
||||
|
||||
volumes:
|
||||
- ./data:/home/data
|
||||
- ../../../docs:/home/docs
|
||||
- ./src/data:/home/data
|
||||
- ./src/docs:/home/docs
|
||||
|
|
|
|||
|
|
@ -1,96 +0,0 @@
|
|||
package main
|
||||
|
||||
import (
|
||||
"flag"
|
||||
"log"
|
||||
"net/http"
|
||||
"strings"
|
||||
"path/filepath"
|
||||
"os"
|
||||
)
|
||||
|
||||
func main() {
|
||||
port := flag.String("p", "8080", "Http listen port")
|
||||
staticDirectory := flag.String("s", "static", "Static directory content")
|
||||
docsDirectory := flag.String("docs", "", "Docs directory content")
|
||||
dataDirectory := flag.String("data", "", "Data directory content")
|
||||
trace := flag.Bool("trace", false, "Produce trace logs")
|
||||
flag.Parse()
|
||||
|
||||
if *trace == true {
|
||||
log.Printf("static folder: %s\n", *staticDirectory)
|
||||
log.Printf("data folder: %s\n", *dataDirectory)
|
||||
log.Printf("docs folder: %s\n", *docsDirectory)
|
||||
}
|
||||
|
||||
http.HandleFunc("/data/", func( res http.ResponseWriter, req *http.Request ) {
|
||||
safePath := ValidatePath(filepath.FromSlash(req.URL.Path[1:]))
|
||||
if (*dataDirectory != "") {
|
||||
safePath = filepath.FromSlash(filepath.Join(*dataDirectory, strings.Replace(safePath, "data/", "", 1)))
|
||||
}
|
||||
http.ServeFile(res, req, safePath)
|
||||
})
|
||||
|
||||
http.HandleFunc("/docs/", func( res http.ResponseWriter, req *http.Request ) {
|
||||
localPath := ""
|
||||
if (*docsDirectory == "") {
|
||||
temp := "../../.."
|
||||
docsDirectory = &temp
|
||||
localPath = ValidatePath(filepath.FromSlash(*docsDirectory+req.URL.Path))
|
||||
} else {
|
||||
localPath = ValidatePath(filepath.FromSlash(*docsDirectory+strings.Replace(req.URL.Path, "docs/", "", 1)))
|
||||
}
|
||||
if *trace == true {
|
||||
log.Printf("fetch docs: \"%s\"", localPath)
|
||||
}
|
||||
f, err := os.Open(localPath)
|
||||
if err != nil {
|
||||
log.Printf("fetch docs error: \"%s\" with %s", localPath, err)
|
||||
http.ServeFile(res, req, filepath.FromSlash(*staticDirectory + "/404.html"));
|
||||
} else {
|
||||
s, err := f.Stat()
|
||||
if err != nil || s.IsDir() {
|
||||
log.Printf("fetch docs stat error: \"%s\"", localPath)
|
||||
http.ServeFile(res, req, filepath.FromSlash(*staticDirectory + "/404.html"))
|
||||
} else {
|
||||
http.ServeFile(res, req, localPath)
|
||||
}
|
||||
}
|
||||
})
|
||||
|
||||
fileServer := http.FileServer(FileSystem{http.Dir(*staticDirectory)})
|
||||
http.Handle("/", fileServer)
|
||||
|
||||
log.Printf("Serving on HTTP port: %s\n", *port)
|
||||
log.Fatal(http.ListenAndServe(":"+*port, nil))
|
||||
}
|
||||
|
||||
|
||||
|
||||
type FileSystem struct {
|
||||
fs http.FileSystem
|
||||
}
|
||||
|
||||
func (fs FileSystem) Open(path string) (http.File, error) {
|
||||
f, err := fs.fs.Open(path)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
s, err := f.Stat()
|
||||
if s.IsDir() {
|
||||
index := strings.TrimSuffix(path, "/") + "/index.html"
|
||||
if _, err := fs.fs.Open(index); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
}
|
||||
|
||||
return f, nil
|
||||
}
|
||||
|
||||
func ValidatePath(path string) string {
|
||||
|
||||
safePath := path
|
||||
|
||||
return safePath
|
||||
}
|
||||
|
|
@ -0,0 +1,171 @@
|
|||
{
|
||||
"id": null,
|
||||
"describe": "BSB file metadata details. The file is a CSV (RFC4180) formatted file.",
|
||||
"encoding": "UTF-8",
|
||||
"format": "RFC4180",
|
||||
"multiRecord": false,
|
||||
"lineEndings": "CRLF",
|
||||
"header": false,
|
||||
"footer": false,
|
||||
"delimiter": ",",
|
||||
"quote": "\"",
|
||||
"escape": "\"",
|
||||
"source": null,
|
||||
|
||||
|
||||
"recordFormat": [
|
||||
{
|
||||
"formatName" : "Data",
|
||||
"columns": [
|
||||
{
|
||||
"columnName": "BSB",
|
||||
"description": "Bank State Branch (BSB) code that is unique",
|
||||
"type": "char",
|
||||
"length": 7,
|
||||
"nullable": false,
|
||||
"default": null,
|
||||
"dataset": "",
|
||||
"formatting": "###-###",
|
||||
"labels": {
|
||||
"leftLabel": "BSB",
|
||||
"rightLabel": "",
|
||||
"heading": "BSB"
|
||||
}
|
||||
},
|
||||
{
|
||||
"columnName": "Bank",
|
||||
"description": "Mnemonic for the bank",
|
||||
"type": "varchar",
|
||||
"length": 3,
|
||||
"nullable": false,
|
||||
"default": null,
|
||||
"dataset": "",
|
||||
"formatting": "",
|
||||
"labels": {
|
||||
"leftLabel": "Bank",
|
||||
"rightLabel": "",
|
||||
"heading": "Bank"
|
||||
}
|
||||
},
|
||||
{
|
||||
"columnName": "Branch",
|
||||
"description": "Branch name and in some cases describes the type of business it conducts",
|
||||
"type": "varchar",
|
||||
"length": 35,
|
||||
"nullable": false,
|
||||
"default": null,
|
||||
"dataset": "",
|
||||
"formatting": "",
|
||||
"labels": {
|
||||
"leftLabel": "Branch",
|
||||
"rightLabel": "",
|
||||
"heading": ""
|
||||
}
|
||||
},
|
||||
{
|
||||
"columnName": "StreetAddress",
|
||||
"description": "Street address for the bank branch. This includes the level, street number and street type",
|
||||
"type": "varchar",
|
||||
"length": 35,
|
||||
"nullable": false,
|
||||
"default": null,
|
||||
"dataset": "",
|
||||
"formatting": "",
|
||||
"labels": {
|
||||
"leftLabel": "Address",
|
||||
"rightLabel": "",
|
||||
"heading": "Street Address"
|
||||
}
|
||||
},
|
||||
{
|
||||
"columnName": "Town",
|
||||
"description": "Town where the branch is located",
|
||||
"type": "varchar",
|
||||
"length": 20,
|
||||
"nullable": false,
|
||||
"default": null,
|
||||
"dataset": "",
|
||||
"formatting": "",
|
||||
"labels": {
|
||||
"leftLabel": "Town/City",
|
||||
"rightLabel": "",
|
||||
"heading": "Town/City"
|
||||
}
|
||||
},
|
||||
{
|
||||
"columnName": "State",
|
||||
"description": "Australian state or terrritory where the branch is located",
|
||||
"type": "varchar",
|
||||
"length": 3,
|
||||
"nullable": false,
|
||||
"default": null,
|
||||
"dataset": "NSW, VIC, QLD, SA, NT, WA, TAS, ACT",
|
||||
"formatting": "",
|
||||
"labels": {
|
||||
"leftLabel": "State",
|
||||
"rightLabel": "",
|
||||
"heading": "State"
|
||||
}
|
||||
},
|
||||
{
|
||||
"columnName": "Postcode",
|
||||
"description": "Australian postcode for the bank branch",
|
||||
"type": "int",
|
||||
"length": 4,
|
||||
"nullable": false,
|
||||
"default": null,
|
||||
"dataset": "",
|
||||
"formatting": "####",
|
||||
"labels": {
|
||||
"leftLabel": "Postcode",
|
||||
"rightLabel": "",
|
||||
"heading": "PostCode"
|
||||
}
|
||||
},
|
||||
{
|
||||
"columnName": "Payments",
|
||||
"description": "Payments flag. The value is a concatenation of P=Paper Payments, E=Electronic Payments, H=High Value Payments",
|
||||
"type": "varchar",
|
||||
"length": 3,
|
||||
"nullable": false,
|
||||
"default": null,
|
||||
"dataset": "P, E, H, PE, PH, EH, PEH",
|
||||
"formatting": "",
|
||||
"labels": {
|
||||
"leftLabel": "Payment Types",
|
||||
"rightLabel": "",
|
||||
"heading": ""
|
||||
}
|
||||
}
|
||||
]
|
||||
}
|
||||
],
|
||||
"indexes": [
|
||||
{
|
||||
"indexName": "BSB",
|
||||
"primary": true,
|
||||
"unique": true,
|
||||
"columns": [
|
||||
{"name": "BSB"}
|
||||
]
|
||||
}
|
||||
],
|
||||
"sample": [
|
||||
{"record": "\"012-028\",\"ANZ\",\"Banking Products NSW Phone\",\"Level 7 75 Dorcas Street\",\"South Melbourne\",\"VIC\",\"3205\",\"PE\""},
|
||||
{"record": "\"033-062\",\"WBC\",\"Moonee Ponds\",\"71 Puckle Street\",\"Moonee Ponds\",\"VIC\",\"3039\",\"PEH\""},
|
||||
{"record": "\"062-731\",\"CBA\",\"Narromine\",\"65 Dandaloo St Cnr Nymagee Street\",\"Narromine\",\"NSW\",\"2821\",\"PEH\""}
|
||||
],
|
||||
"custom": [
|
||||
{
|
||||
"extension": "software",
|
||||
"softwareName": "MartiReference",
|
||||
"author": "Meerkat@merebox.com",
|
||||
"version": "0.0.1"
|
||||
},
|
||||
{
|
||||
"extension": "template",
|
||||
"renderer": "MARTILQREFERENCE:Mustache",
|
||||
"url": "template/details/martilq_cols_csv.must"
|
||||
}
|
||||
]
|
||||
}
|
||||
|
|
@ -0,0 +1,311 @@
|
|||
{
|
||||
"id": null,
|
||||
"describe": "BSB file metadata details. The file is a legacy fixed column position file. The record length is not fixed as the last column can be stripped of blanks. The length is 112 to 113 bytes. Position are 1 index based",
|
||||
"encoding": "UTF-8",
|
||||
"format": "Fixed Column Position (FCP)",
|
||||
"multiRecord": true,
|
||||
"lineEndings": "CRLF",
|
||||
"header": true,
|
||||
"footer": true,
|
||||
"source": null,
|
||||
|
||||
"recordFormat": [
|
||||
{
|
||||
"formatName" : "Header",
|
||||
"columns": [
|
||||
{
|
||||
"columnName": "TXT1",
|
||||
"description": "Fixed identifier of \"HEADER RECORD Effective Date: \"",
|
||||
"type": "char",
|
||||
"starting": 1,
|
||||
"ending": 31,
|
||||
"dataset": "",
|
||||
"formatting": "",
|
||||
"labels": {
|
||||
"leftLabel": "TXT1",
|
||||
"rightLabel": "",
|
||||
"heading": "TXT1"
|
||||
}
|
||||
},
|
||||
{
|
||||
"columnName": "EffectiveDate",
|
||||
"description": "Formatted effective date",
|
||||
"type": "char",
|
||||
"starting": 32,
|
||||
"ending": 42,
|
||||
"dataset": "",
|
||||
"formatting": "DD mmm YYYY",
|
||||
"labels": {
|
||||
"leftLabel": "EffectiveDate",
|
||||
"rightLabel": "",
|
||||
"heading": "EffectiveDate"
|
||||
}
|
||||
},
|
||||
{
|
||||
"columnName": "TXT2",
|
||||
"description": "Fixed identifier of \" File Created At: \"",
|
||||
"type": "char",
|
||||
"starting": 43,
|
||||
"ending": 64,
|
||||
"dataset": "",
|
||||
"formatting": "",
|
||||
"labels": {
|
||||
"leftLabel": "TXT2",
|
||||
"rightLabel": "",
|
||||
"heading": "TXT2"
|
||||
}
|
||||
},
|
||||
{
|
||||
"columnName": "RunDate",
|
||||
"description": "Formatted run date",
|
||||
"type": "char",
|
||||
"starting": 65,
|
||||
"ending": 84,
|
||||
"dataset": "",
|
||||
"formatting": "DD mmm YYYY HH:mm:ss",
|
||||
"labels": {
|
||||
"leftLabel": "EffectiveDate",
|
||||
"rightLabel": "",
|
||||
"heading": "EffectiveDate"
|
||||
}
|
||||
},
|
||||
{
|
||||
"columnName": "TXT3",
|
||||
"description": "Fixed identifier of \" Report Number: \"",
|
||||
"type": "char",
|
||||
"starting": 85,
|
||||
"ending": 103,
|
||||
"dataset": "",
|
||||
"formatting": "",
|
||||
"labels": {
|
||||
"leftLabel": "TXT3",
|
||||
"rightLabel": "",
|
||||
"heading": "TXT3"
|
||||
}
|
||||
},
|
||||
{
|
||||
"columnName": "ReportNumber",
|
||||
"description": "Report sequential number",
|
||||
"type": "int",
|
||||
"starting": 104,
|
||||
"ending": 105,
|
||||
"dataset": "",
|
||||
"formatting": "###",
|
||||
"labels": {
|
||||
"leftLabel": "Report#",
|
||||
"rightLabel": "",
|
||||
"heading": "Report#"
|
||||
}
|
||||
},
|
||||
{
|
||||
"columnName": "TXT4",
|
||||
"description": "Trailing blanks for padding",
|
||||
"type": "char",
|
||||
"starting": 106,
|
||||
"ending": 111,
|
||||
"dataset": "",
|
||||
"formatting": "",
|
||||
"labels": {
|
||||
"leftLabel": "TXT4",
|
||||
"rightLabel": "",
|
||||
"heading": "TXT4"
|
||||
}
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"formatName" : "Data",
|
||||
"columns": [
|
||||
{
|
||||
"columnName": "BSB",
|
||||
"description": "Bank State Branch (BSB) code that is unique",
|
||||
"type": "char",
|
||||
"starting": 1,
|
||||
"ending": 7,
|
||||
"dataset": "",
|
||||
"formatting": "###-###",
|
||||
"labels": {
|
||||
"leftLabel": "BSB",
|
||||
"rightLabel": "",
|
||||
"heading": "BSB"
|
||||
}
|
||||
},
|
||||
{
|
||||
"columnName": "Bank",
|
||||
"description": "Mnemonic for the bank",
|
||||
"type": "varchar",
|
||||
"starting": 8,
|
||||
"ending": 10,
|
||||
"dataset": "",
|
||||
"formatting": "",
|
||||
"labels": {
|
||||
"leftLabel": "Bank",
|
||||
"rightLabel": "",
|
||||
"heading": "Bank"
|
||||
}
|
||||
},
|
||||
{
|
||||
"columnName": "Branch",
|
||||
"description": "Branch name and in some cases describes the type of business it conducts",
|
||||
"type": "varchar",
|
||||
"starting": 11,
|
||||
"ending": 45,
|
||||
"dataset": "",
|
||||
"formatting": "",
|
||||
"labels": {
|
||||
"leftLabel": "Branch",
|
||||
"rightLabel": "",
|
||||
"heading": ""
|
||||
}
|
||||
},
|
||||
{
|
||||
"columnName": "StreetAddress",
|
||||
"description": "Street address for the bank branch. This includes the level, street number and street type",
|
||||
"type": "varchar",
|
||||
"starting": 46,
|
||||
"ending": 80,
|
||||
"dataset": "",
|
||||
"formatting": "",
|
||||
"labels": {
|
||||
"leftLabel": "Address",
|
||||
"rightLabel": "",
|
||||
"heading": "Street Address"
|
||||
}
|
||||
},
|
||||
{
|
||||
"columnName": "Town",
|
||||
"description": "Town where the branch is located",
|
||||
"type": "varchar",
|
||||
"starting": 81,
|
||||
"ending": 100,
|
||||
"dataset": "",
|
||||
"formatting": "",
|
||||
"labels": {
|
||||
"leftLabel": "Town/City",
|
||||
"rightLabel": "",
|
||||
"heading": "Town/City"
|
||||
}
|
||||
},
|
||||
{
|
||||
"columnName": "State",
|
||||
"description": "Australian state or terrritory where the branch is located",
|
||||
"type": "varchar",
|
||||
"starting": 101,
|
||||
"ending": 103,
|
||||
"dataset": "NSW, VIC, QLD, SA, NT, WA, TAS, ACT",
|
||||
"formatting": "",
|
||||
"labels": {
|
||||
"leftLabel": "State",
|
||||
"rightLabel": "",
|
||||
"heading": "State"
|
||||
}
|
||||
},
|
||||
{
|
||||
"columnName": "Postcode",
|
||||
"description": "Australian postcode for the bank branch",
|
||||
"type": "int",
|
||||
"starting": 104,
|
||||
"ending": 107,
|
||||
"dataset": "",
|
||||
"formatting": "####",
|
||||
"labels": {
|
||||
"leftLabel": "Postcode",
|
||||
"rightLabel": "",
|
||||
"heading": "PostCode"
|
||||
}
|
||||
},
|
||||
{
|
||||
"columnName": "Payments",
|
||||
"description": "Payments flag. The value is a concatenation of P=Paper Payments, E=Electronic Payments, H=High Value Payments",
|
||||
"type": "varchar",
|
||||
"starting": 108,
|
||||
"ending": 111,
|
||||
"dataset": "P, E, H, PE, PH, EH, PEH",
|
||||
"formatting": "",
|
||||
"labels": {
|
||||
"leftLabel": "Payment Types",
|
||||
"rightLabel": "",
|
||||
"heading": ""
|
||||
}
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"formatName" : "Footer",
|
||||
"columns": [
|
||||
{
|
||||
"columnName": "TXT5",
|
||||
"description": "Fixed string \"TRAILER RECORD Total number of records (except header and trailer): \"",
|
||||
"type": "char",
|
||||
"starting": 1,
|
||||
"ending": 72,
|
||||
"dataset": "",
|
||||
"formatting": "",
|
||||
"labels": {
|
||||
"leftLabel": "TXT5",
|
||||
"rightLabel": "",
|
||||
"heading": "TXT5"
|
||||
}
|
||||
},
|
||||
{
|
||||
"columnName": "RecordCount",
|
||||
"description": "Number of data records in the file",
|
||||
"type": "int",
|
||||
"starting": 73,
|
||||
"ending": 78,
|
||||
"dataset": "",
|
||||
"formatting": "##,###",
|
||||
"labels": {
|
||||
"leftLabel": "Report#",
|
||||
"rightLabel": "",
|
||||
"heading": "Report#"
|
||||
}
|
||||
},
|
||||
{
|
||||
"columnName": "TXT6",
|
||||
"description": "Trailing blanks for padding",
|
||||
"type": "char",
|
||||
"starting": 79,
|
||||
"ending": 112,
|
||||
"dataset": "",
|
||||
"formatting": "",
|
||||
"labels": {
|
||||
"leftLabel": "TXT6",
|
||||
"rightLabel": "",
|
||||
"heading": "TXT6"
|
||||
}
|
||||
}
|
||||
]
|
||||
}
|
||||
],
|
||||
"indexes": [
|
||||
{
|
||||
"indexName": "BSB",
|
||||
"primary": true,
|
||||
"unique": true,
|
||||
"columns": [
|
||||
{"name": "BSB"}
|
||||
]
|
||||
}
|
||||
],
|
||||
"sample": [
|
||||
{"record": "HEADER RECORD Effective Date: 01 Sep 2021 File Created At: 01 Sep 2021 08:58:50 Report Number: 305 "},
|
||||
{"record": "012-298ANZFrenchs Forest Shop 5, Forestway Shopping Centre Frenchs Forest NSW2086PEH "},
|
||||
{"record": "033-652WBCSwan Hill 169-171 Campbell Street Swan Hill VIC3585PEH "},
|
||||
{"record": "932-000RABRegional Australia Bank Suite 4 Technology Park MadgwickDr Armidale NSW2350PEH "},
|
||||
{"record": "TRAILER RECORD Total number of records (except header and trailer): 15,055 "}
|
||||
],
|
||||
"custom": [
|
||||
{
|
||||
"extension": "software",
|
||||
"softwareName": "MartiReference",
|
||||
"author": "Meerkat@merebox.com",
|
||||
"version": "0.0.1"
|
||||
},
|
||||
{
|
||||
"extension": "template",
|
||||
"renderer": "MARTILQREFERENCE:Mustache",
|
||||
"url": "template/details/martilq_cols_fcp.must"
|
||||
}
|
||||
]
|
||||
}
|
||||
|
|
@ -1,5 +1,5 @@
|
|||
{
|
||||
"content-type": "application/vnd.martilq.json",
|
||||
"contentType": "application/vnd.martilq.json",
|
||||
"title": "Conversion from CKAN",
|
||||
"uid": "ab7eddce-84df-4098-bc8f-500d0d9776d1",
|
||||
"description": "This data has been converted from DATA GOV AU CKAN data source with URL 'https://data.gov.au/api/3/action/package_show?id=ab7eddce-84df-4098-bc8f-500d0d9776d1'",
|
||||
|
|
@ -139,8 +139,8 @@
|
|||
},
|
||||
{
|
||||
"extension": "template",
|
||||
"renderer": "MartiReference:Mustache",
|
||||
"url": "data/martilq_ckan.must"
|
||||
"renderer": "MARTILQREFERENCE:Mustache",
|
||||
"url": "template/martilq_ckan.must"
|
||||
}
|
||||
]
|
||||
}
|
||||
|
|
@ -1,5 +1,5 @@
|
|||
{
|
||||
"content-type": "application/vnd.martilq.json",
|
||||
"contentType": "application/vnd.martilq.json",
|
||||
"title": "Conversion from CKAN",
|
||||
"uid": "f2b7c2c1-f4ef-4ae9-aba5-45c19e4d3038",
|
||||
"description": "###Update November 2019 - additional fields ###\r\n\r\nFrom 21 November 2019, the dataset will be updated to include 7 new fields (see help file for details)\r\n\r\nThese fields are included in conjunction with the professional standards reforms for financial advisers. More information can be found on the ASIC website https://asic.gov.au/regulatory-resources/financial-services/professional-standards-for-financial-advisers-reforms/.\r\n\r\n__Note:__ For most advisers the new fields will be unpopulated on 21 November 2019. As advisers provide this data to ASIC it will appear in the dataset.\r\n\r\n***\r\n\r\n###Dataset summary###\r\n\r\nASIC is Australia’s corporate, markets and financial services regulator. ASIC contributes to Australia’s economic reputation and wellbeing by ensuring that Australia’s financial markets are fair and transparent, supported by confident and informed investors and consumers. \r\n \r\nAustralian Financial Services Licensees are required to keep the details of their financial advisers up to date on ASIC's Financial Advisers Register. Information contained in the register is made available to the public to search via ASIC's Moneysmart website. \r\n\r\nSelect data from the Financial Advisers Register will be uploaded each week to www.data.gov.au. The data made available will be a snapshot of the register at a point in time. Legislation prescribes the type of information ASIC is allowed to disclose to the public. \r\n\r\nThe information included in the downloadable dataset is: \r\n\r\n* Adviser name\r\n* Adviser number\r\n* Adviser role\r\n* Adviser sub type\r\n* Adviser role status\r\n* Adviser ABN\r\n* Year first provided advice \r\n* Licence name\r\n* Licence number\r\n* Licence ABN\r\n* Licence controlled by\r\n* Adviser start date\r\n* Adviser end date\r\n* Adviser CPD failure year\r\n* Adviser principal business address suburb\r\n* Adviser principal business address State/Territory\r\n* Adviser principal business address postcode\r\n* Adviser principal business address Country\r\n* Appointing representative name\r\n* Appointing representative number\r\n* Appointing representative ABN\r\n* Disciplinary action start date\r\n* Disciplinary action end date\r\n* Disciplinary action type\r\n* Product authorisations (for a full list see the Financial Adviser Register – Help File)\r\n* Qualifications and Training\r\n* FASEA approved qualifications\r\n* Memberships\r\n* Further restrictions\r\n\r\nAdditional information about financial advisers can be found via [ASIC's website] (http://www.asic.gov.au/ \"ASIC's website\"). Accessing some information may attract a fee. \r\n\r\nMore information about searching [ASIC's registers] (http://www.asic.gov.au/online-services/search-asics-registers/ \"ASIC's registers\"). ",
|
||||
|
|
@ -94,8 +94,8 @@
|
|||
},
|
||||
{
|
||||
"extension": "template",
|
||||
"renderer": "MartiReference:Mustache",
|
||||
"url": "data/martilq_ckan.must"
|
||||
"renderer": "MARTILQREFERENCE:Mustache",
|
||||
"url": "template/martilq_ckan.must"
|
||||
}
|
||||
]
|
||||
}
|
||||
|
|
@ -1,14 +1,15 @@
|
|||
{
|
||||
"content-type": "application/vnd.martilq.json",
|
||||
"contentType": "application/vnd.martilq.json",
|
||||
"title": "Australian BSB source",
|
||||
"uid": "ae5d4d4b-5a7f-494d-8629-a0ced5bc7ba0",
|
||||
"description": "The original files are published on the APN FTP site and have been copied to a web server",
|
||||
"issued": "2021-11-24",
|
||||
"modified": "2021-11-05",
|
||||
"publisher": "Australian Payment Network",
|
||||
"contactPoint": "",
|
||||
"contactPoint": "operations@auspaynet.com.au",
|
||||
"accessLevel": "Confidential",
|
||||
"rights": "Restricted",
|
||||
"tags": [],
|
||||
"tags": ["payment", "au", "bank"],
|
||||
"license": "",
|
||||
"state": "active",
|
||||
"batch": 1.0,
|
||||
|
|
@ -22,7 +23,7 @@
|
|||
"documentName": "BSBDirectoryOct21-307.csv",
|
||||
"issuedDate": "2021-11-05T20:58:03",
|
||||
"modified": "2021-11-01T09:00:45",
|
||||
"expires": "",
|
||||
"expires": "2021-12-01T09:00:45",
|
||||
"state": "active",
|
||||
"author": "Meerkat@merebox.com",
|
||||
"length": 1352205,
|
||||
|
|
@ -32,9 +33,10 @@
|
|||
"signed": false
|
||||
},
|
||||
"description": "",
|
||||
"url": "",
|
||||
"version": "",
|
||||
"content-type": "text/csv",
|
||||
"url": "http://apnedata.merebox.com.s3.ap-southeast-2.amazonaws.com/au/bsb/BSBDirectoryOct21-307.csv",
|
||||
"structure": "/view.html?martilq=details/bsb_details_csv.json",
|
||||
"version": "2012April",
|
||||
"contentType": "text/csv",
|
||||
"encoding": null,
|
||||
"compression": null,
|
||||
"encryption": null,
|
||||
|
|
@ -83,14 +85,68 @@
|
|||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"title": "BSBDirectoryOct21-307",
|
||||
"uid": "0c2ee577-3fbb-46b9-bcc6-40eae069569c",
|
||||
"documentName": "BSBDirectoryOct21-307.txt",
|
||||
"issuedDate": "2021-11-05T20:58:03",
|
||||
"modified": "2021-11-01T09:00:45",
|
||||
"expires": "2021-12-01T09:00:45",
|
||||
"state": "active",
|
||||
"author": "Meerkat@merebox.com",
|
||||
"length": 1352205,
|
||||
"hash": {
|
||||
"algo": "SHA256",
|
||||
"value": "dd01585bed7a0b6da5d4deb5dff70a4b8ccfee873cf0454f2a46910c3275f03e",
|
||||
"signed": false
|
||||
},
|
||||
"description": "",
|
||||
"url": "http://apnedata.merebox.com.s3.ap-southeast-2.amazonaws.com/au/bsb/BSBDirectoryOct21-307.txt",
|
||||
"structure": "/view.html?martilq=details/bsb_details_txt.json",
|
||||
"version": "2012April",
|
||||
"contentType": "text/txt",
|
||||
"encoding": null,
|
||||
"compression": null,
|
||||
"encryption": null,
|
||||
"attributes": [
|
||||
{
|
||||
"category": "dataset",
|
||||
"name": "header",
|
||||
"function": "count",
|
||||
"comparison": "EQ",
|
||||
"value": 1
|
||||
},
|
||||
{
|
||||
"category": "dataset",
|
||||
"name": "footer",
|
||||
"function": "count",
|
||||
"comparison": "EQ",
|
||||
"value": 1
|
||||
},
|
||||
{
|
||||
"category": "dataset",
|
||||
"name": "records",
|
||||
"function": "count",
|
||||
"comparison": "EQ",
|
||||
"value": 15054
|
||||
},
|
||||
{
|
||||
"category": "dataset",
|
||||
"name": "columns",
|
||||
"function": "count",
|
||||
"comparison": "EQ",
|
||||
"value": 8
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"title": "BSBDirectorySep21-306",
|
||||
"uid": "900e815a-e052-4198-9d92-e0f1aac07616",
|
||||
"documentName": "BSBDirectorySep21-306.csv",
|
||||
"issuedDate": "2021-11-05T20:58:03",
|
||||
"modified": "2021-10-01T09:00:48",
|
||||
"expires": "",
|
||||
"state": "active",
|
||||
"expires": "2021-11-01T09:00:48",
|
||||
"state": "expired",
|
||||
"author": "Meerkat@merebox.com",
|
||||
"length": 1351281,
|
||||
"hash": {
|
||||
|
|
@ -99,9 +155,10 @@
|
|||
"signed": false
|
||||
},
|
||||
"description": "",
|
||||
"url": "",
|
||||
"version": "",
|
||||
"content-type": "text/csv",
|
||||
"url": "http://apnedata.merebox.com.s3.ap-southeast-2.amazonaws.com/au/bsb/BSBDirectorySep21-306.csv",
|
||||
"structure": "/view.html?martilq=details/bsb_details_csv.json",
|
||||
"version": "2012April",
|
||||
"contentType": "text/csv",
|
||||
"encoding": null,
|
||||
"compression": null,
|
||||
"encryption": null,
|
||||
|
|
@ -156,8 +213,8 @@
|
|||
"documentName": "BSBDirectoryAug21-305.csv",
|
||||
"issuedDate": "2021-11-05T20:58:03",
|
||||
"modified": "2021-09-01T09:00:43",
|
||||
"expires": "",
|
||||
"state": "active",
|
||||
"expires": "2021-10-01T09:00:43",
|
||||
"state": "expired",
|
||||
"author": "Meerkat@merebox.com",
|
||||
"length": 1352016,
|
||||
"hash": {
|
||||
|
|
@ -166,9 +223,10 @@
|
|||
"signed": false
|
||||
},
|
||||
"description": "",
|
||||
"url": "",
|
||||
"version": "",
|
||||
"content-type": "text/csv",
|
||||
"url": "http://apnedata.merebox.com.s3.ap-southeast-2.amazonaws.com/au/bsb/BSBDirectoryAug21-305.csv",
|
||||
"structure": "/view.html?martilq=details/bsb_details_csv.json",
|
||||
"version": "2012April",
|
||||
"contentType": "text/csv",
|
||||
"encoding": null,
|
||||
"compression": null,
|
||||
"encryption": null,
|
||||
|
|
@ -223,8 +281,8 @@
|
|||
"documentName": "BSBDirectoryJul21-304.csv",
|
||||
"issuedDate": "2021-11-05T20:58:03",
|
||||
"modified": "2021-08-05T09:00:46",
|
||||
"expires": "",
|
||||
"state": "active",
|
||||
"expires": "2021-09-05T09:00:46",
|
||||
"state": "expired",
|
||||
"author": "Meerkat@merebox.com",
|
||||
"length": 1351065,
|
||||
"hash": {
|
||||
|
|
@ -233,9 +291,10 @@
|
|||
"signed": false
|
||||
},
|
||||
"description": "",
|
||||
"url": "",
|
||||
"version": "",
|
||||
"content-type": "text/csv",
|
||||
"url": "http://apnedata.merebox.com.s3.ap-southeast-2.amazonaws.com/au/bsb/BSBDirectoryJul21-304.csv",
|
||||
"structure": "/view.html?martilq=details/bsb_details_csv.json",
|
||||
"version": "2012April",
|
||||
"contentType": "text/csv",
|
||||
"encoding": null,
|
||||
"compression": null,
|
||||
"encryption": null,
|
||||
|
|
@ -290,8 +349,8 @@
|
|||
"documentName": "BSBDirectoryJun21-303.csv",
|
||||
"issuedDate": "2021-11-05T20:58:03",
|
||||
"modified": "2021-07-01T09:00:48",
|
||||
"expires": "",
|
||||
"state": "active",
|
||||
"expires": "2021-08-01T09:00:48",
|
||||
"state": "expired",
|
||||
"author": "Meerkat@merebox.com",
|
||||
"length": 1350547,
|
||||
"hash": {
|
||||
|
|
@ -300,9 +359,10 @@
|
|||
"signed": false
|
||||
},
|
||||
"description": "",
|
||||
"url": "",
|
||||
"version": "",
|
||||
"content-type": "text/csv",
|
||||
"url": "http://apnedata.merebox.com.s3.ap-southeast-2.amazonaws.com/au/bsb/BSBDirectoryJun21-303.csv",
|
||||
"structure": "/view.html?martilq=details/bsb_details_csv.json",
|
||||
"version": "2012April",
|
||||
"contentType": "text/csv",
|
||||
"encoding": null,
|
||||
"compression": null,
|
||||
"encryption": null,
|
||||
|
|
@ -357,8 +417,8 @@
|
|||
"documentName": "BSBDirectoryMay21-302.csv",
|
||||
"issuedDate": "2021-11-05T20:58:03",
|
||||
"modified": "2021-06-01T09:00:52",
|
||||
"expires": "",
|
||||
"state": "active",
|
||||
"expires": "2021-07-01T09:00:52",
|
||||
"state": "expired",
|
||||
"author": "Meerkat@merebox.com",
|
||||
"length": 1350255,
|
||||
"hash": {
|
||||
|
|
@ -367,9 +427,10 @@
|
|||
"signed": false
|
||||
},
|
||||
"description": "",
|
||||
"url": "",
|
||||
"version": "",
|
||||
"content-type": "text/csv",
|
||||
"url": "http://apnedata.merebox.com.s3.ap-southeast-2.amazonaws.com/au/bsb/BSBDirectoryMay21-302.csv",
|
||||
"structure": "/view.html?martilq=details/bsb_details_csv.json",
|
||||
"version": "2012April",
|
||||
"contentType": "text/csv",
|
||||
"encoding": null,
|
||||
"compression": null,
|
||||
"encryption": null,
|
||||
|
|
@ -424,8 +485,8 @@
|
|||
"documentName": "BSBDirectoryApr21-301.csv",
|
||||
"issuedDate": "2021-11-05T20:58:03",
|
||||
"modified": "2021-05-03T09:00:51",
|
||||
"expires": "",
|
||||
"state": "active",
|
||||
"expires": "2021-06-03T09:00:51",
|
||||
"state": "expired",
|
||||
"author": "Meerkat@merebox.com",
|
||||
"length": 1352037,
|
||||
"hash": {
|
||||
|
|
@ -434,9 +495,10 @@
|
|||
"signed": false
|
||||
},
|
||||
"description": "",
|
||||
"url": "",
|
||||
"version": "",
|
||||
"content-type": "text/csv",
|
||||
"url": "http://apnedata.merebox.com.s3.ap-southeast-2.amazonaws.com/au/bsb/BSBDirectoryApr21-301.csv",
|
||||
"structure": "/view.html?martilq=details/bsb_details_csv.json",
|
||||
"version": "2012April",
|
||||
"contentType": "text/csv",
|
||||
"encoding": null,
|
||||
"compression": null,
|
||||
"encryption": null,
|
||||
|
|
@ -491,8 +553,8 @@
|
|||
"documentName": "BSBDirectoryMar21-300.csv",
|
||||
"issuedDate": "2021-11-05T20:58:03",
|
||||
"modified": "2021-04-01T09:00:55",
|
||||
"expires": "",
|
||||
"state": "active",
|
||||
"expires": "2021-05-01T09:00:55",
|
||||
"state": "expired",
|
||||
"author": "Meerkat@merebox.com",
|
||||
"length": 1353104,
|
||||
"hash": {
|
||||
|
|
@ -501,9 +563,10 @@
|
|||
"signed": false
|
||||
},
|
||||
"description": "",
|
||||
"url": "",
|
||||
"version": "",
|
||||
"content-type": "text/csv",
|
||||
"url": "http://apnedata.merebox.com.s3.ap-southeast-2.amazonaws.com/au/bsb/BSBDirectoryMar21-300.csv",
|
||||
"structure": "/view.html?martilq=details/bsb_details_csv.json",
|
||||
"version": "2012April",
|
||||
"contentType": "text/csv",
|
||||
"encoding": null,
|
||||
"compression": null,
|
||||
"encryption": null,
|
||||
|
|
@ -559,6 +622,11 @@
|
|||
"softwareName": "MARTILQREFERENCE",
|
||||
"author": "Meerkat@merebox.com",
|
||||
"version": "0.0.1"
|
||||
},
|
||||
{
|
||||
"extension": "template",
|
||||
"renderer": "MARTILQREFERENCE:Mustache",
|
||||
"url": "template/martilq_default.must"
|
||||
}
|
||||
]
|
||||
}
|
||||
|
|
@ -1,6 +1,6 @@
|
|||
{
|
||||
"content-type": "application/vnd.martilq.json",
|
||||
"title": "Sample martiLQ generatd framework file",
|
||||
"contentType": "application/vnd.martilq.json",
|
||||
"title": "Sample martiLQ generated framework file",
|
||||
"uid": "c525f5c5-ce87-4688-aa36-d0aa59d1e939",
|
||||
"description": "Sample execution of martiLQ execution",
|
||||
"issued": "2021-11-21T16:57:37",
|
||||
|
|
@ -5564,6 +5564,11 @@
|
|||
"softwareName": "MartiReference",
|
||||
"author": "Meerkat@merebox.com",
|
||||
"version": "0.0.1"
|
||||
},
|
||||
{
|
||||
"extension": "template",
|
||||
"renderer": "MARTILQREFERENCE:Mustache",
|
||||
"url": "template/martilq_no_struct.must"
|
||||
}
|
||||
]
|
||||
}
|
||||
|
|
@ -0,0 +1,43 @@
|
|||
# Documentation
|
||||
|
||||
**martiLQ** stands for metadata reconcilation for transfer information, load quality
|
||||
|
||||
Before starting with **martiLQ** it is advisable to understand if it is right for
|
||||
your organisation's needs. Information is available in a number of short
|
||||
documents.
|
||||
|
||||
There is no quickstart document to get you started as each use case and
|
||||
organisation is different. There are sample implementations which you
|
||||
can adjust if they resonate with your circumstances,
|
||||
see [sample implementations](samples/)
|
||||
|
||||
|
||||
There are sample implementations which you
|
||||
can adjust if they resonate with your circumstances.
|
||||
|
||||
The source, documentation and samples are available at `<https://github.com/meerkat-manor/marti>`_
|
||||
|
||||
## MartiLQ objective
|
||||
|
||||
The objective of **martiLQ** is to define a simple standard for
|
||||
capturing the data files being transferred. It is not for
|
||||
real time web service transactions.
|
||||
|
||||
**martiLQ** is about file and document transfer and reconciling
|
||||
that the all files have arrived and have not changed, and if so
|
||||
required are also encrypted.
|
||||
|
||||
The proposition is to have a common, machine readable format
|
||||
for file exchange that:
|
||||
|
||||
* ensures data load quality and reconciles
|
||||
* can be used on Linux or Windows or Mac
|
||||
* can be used with Python, Java, PowerShell, Golang, etc
|
||||
* can be used by web services
|
||||
* uses a text based format (JSON)
|
||||
* can form part of the data processing pipeline
|
||||
|
||||
And finally is easy to understand.
|
||||
|
||||
To get a better understanding have a look at the definition
|
||||
in [martiLQ](martiLQ.md)
|
||||
|
|
@ -0,0 +1,36 @@
|
|||
# Attribute definition
|
||||
|
||||
A Resource can list attributes related to the document / file.
|
||||
|
||||
An attribute is a generic definition and conventions are
|
||||
observed in the definitions that are captured here. The attribute
|
||||
section is where load quality metrics are defined.
|
||||
|
||||
## Attribute definition
|
||||
|
||||
The Attribute is described by the table below. Recommended
|
||||
values are listed but custom values can also be defined, just be
|
||||
certain the recipient is able to understand them.
|
||||
|
||||
Name|Description|Values or Default
|
||||
---|---|---
|
||||
category|A type of attribute|dataset, format
|
||||
name|A name for the attribute|records,columns,header,footer,separator,quote, escape
|
||||
function|A function to perform|count,sum,unique
|
||||
comparison|A comparison value or NA|NA, EQ, NE, GT, GE, LT,LE
|
||||
value|The value for the attribute based on the above complex key, excluding comparison|numeric
|
||||
|
||||
A sample JSON is shown below which describes the
|
||||
number of records in the file for the given format.
|
||||
|
||||
```json
|
||||
"attributes": [
|
||||
{
|
||||
"category": "dataset",
|
||||
"name": "records",
|
||||
"function": "count",
|
||||
"comparison": "EQ",
|
||||
"value": "9"
|
||||
}
|
||||
]
|
||||
```
|
||||
|
|
@ -0,0 +1,15 @@
|
|||
# CKAN definition
|
||||
|
||||
The **martiLQ** has used similar terms and structures that are found in the
|
||||
CKAN API describing the resources. This similarity allows for simple mapping of values
|
||||
from the CKAN format to **martiLQ** format.
|
||||
|
||||
What **martiLQ** extends above the CKAN definition is attributes that can be
|
||||
used to reconcile with the received data, plus the ability to define
|
||||
compressed and encrypted resources.
|
||||
|
||||
For more information on CKAN see https://ckan.org/
|
||||
|
||||
A sample JSON to compare against the **martiLQ** document
|
||||
is https://data.gov.au/data/dataset/f2b7c2c1-f4ef-4ae9-aba5-45c19e4d3038
|
||||
|
||||
|
|
@ -0,0 +1,46 @@
|
|||
Comparison of martiLQ document definition
|
||||
=========================================
|
||||
|
||||
The use of metadata definitions is not unique and examples
|
||||
exist in many different situations. Some are standard and open
|
||||
while others are closed.
|
||||
|
||||
Some open standards are EXIF data for pictures, SQL DDL defintions
|
||||
for databases, the XMP definition and web header responses before the
|
||||
web content.
|
||||
|
||||
The **martiLQ** document definition is intended to cover the situation
|
||||
where data files are being transferred and reconciliation is required.
|
||||
|
||||
The **martiLQ** document definition is modelled on
|
||||
the [CKAN API metadata](https://docs.ckan.org/en/2.9/api/index.html)
|
||||
which has been adapted to included additional elements relevant to when
|
||||
you are exchanging data files. This includes the reconciliation elements
|
||||
such as number of records and file hash.
|
||||
|
||||
As the definition is based on the CKAN API, there are tools to import
|
||||
a CKAN source into a **martiLQ** document definition and then process the data
|
||||
through the pipeline as you would for any other data file that had a
|
||||
**martiLQ** document definition.
|
||||
|
||||
Benefit of CKAN and martiLQ
|
||||
---------------------------
|
||||
|
||||
The CKAN is excellent at defining the data source details but it lacks information
|
||||
for load quality. If you have CKAN deployed in your organisation and wish
|
||||
exhange or process the data referenced in CKAN, then there are synergies between
|
||||
CKAN and marti.
|
||||
|
||||
Samples exist on CKAN integration.
|
||||
|
||||
Magda and martiLQ
|
||||
-----------------
|
||||
|
||||
Another source of data is [Magda](https://magda.io/) which has API metadata
|
||||
definitions. Magda is more about data federation and as such provides
|
||||
functionality on finding data sources and describing the contents.
|
||||
|
||||
The Magda software is able to generate APIs and data content. This does not
|
||||
address the needs of data processing pipeline when reconciliation is required.
|
||||
|
||||
If you have Magda data sources then synergies exist between Magda and martiLQ.
|
||||
|
|
@ -0,0 +1,10 @@
|
|||
Custom Definition
|
||||
=================
|
||||
|
||||
The custome definition section allows the inclusion of extensions
|
||||
to the standard. To demonstrate the inclusion, there are three
|
||||
sample extensions. These are:
|
||||
|
||||
* Software - describing the **martiLQ** software version
|
||||
* Spatial - Defining the geographical boundary of the documents
|
||||
* Temporal - Defining date and time aspects of the documents
|
||||
|
|
@ -0,0 +1,8 @@
|
|||
Magda definitions
|
||||
=================
|
||||
|
||||
https://magda.io/
|
||||
|
||||
https://search.data.gov.au/api/v0/apidocs/index.html
|
||||
|
||||
|
||||
|
|
@ -0,0 +1,172 @@
|
|||
# MartiLQ document
|
||||
|
||||
The metadata reconciliation transfer information is referred
|
||||
to as the **martiLQ** document throughout this documentation.
|
||||
|
||||
The **martiLQ** document can be part of a message or a file
|
||||
in its own right. The definition is currently a JSON file.
|
||||
|
||||
## Structure
|
||||
|
||||
The JSON is composed of:
|
||||
|
||||
* A root definition that contains information applicable to all reosurces
|
||||
* A [resource](resources.md) list that contains information related
|
||||
to each document or file
|
||||
* [Attribute](attributes.md) list as a child to each resource
|
||||
* A [custom](custom.md) list
|
||||
|
||||
## Mandatory root information
|
||||
|
||||
The mandatory information in the root of the **martiLQ** document is:
|
||||
|
||||
Name|Description|Default or values
|
||||
---|---|---
|
||||
title|A title or batch identifier. Use a name that is easy to understand or relate to|None
|
||||
uid|Unique identifier for the document. If the same document is reproduced this may not change but the minor number of the batch must change|Auto generated
|
||||
resources|Resource list. See Resource section summary below or detailed document [Resource](resources.md)|At least one required
|
||||
|
||||
### Optional information
|
||||
|
||||
|
||||
The optional information is described in the table below. A number of elements can be configured for their
|
||||
default value(s) in a configuration file.
|
||||
|
||||
Name|Description|Default or values
|
||||
---|---|--
|
||||
description|Long description of the purpose, background or files included|None
|
||||
modified|Modified date and time of the **martiLQ** document|Now
|
||||
tags|List of tags or keywords|
|
||||
publisher|Publisher name|
|
||||
contactPoint|Contact point of a person or team|
|
||||
accessLevel|Acces level|
|
||||
rights|Rights|
|
||||
|
||||
* Batch
|
||||
* License
|
||||
* Described By - A link to the metadata describing the document.
|
||||
More detailed information could be supplied at the link
|
||||
* Landing page
|
||||
* Theme
|
||||
* Custom list - List of custom entries, one being the **martiLQ** software details
|
||||
see [custom](custom.md)
|
||||
|
||||
### Information extension
|
||||
|
||||
The information supplied can be extended by party agreement and there
|
||||
are place holders in the defintion.
|
||||
|
||||
## Resource
|
||||
|
||||
The resource section is a list of documents or files that are to be grouped
|
||||
together are listed under the same **martiLQ** definition.
|
||||
|
||||
At least one document or file must be included. If the same resource is repeated
|
||||
it will commonly be for definiting multiple formats, with each file having a
|
||||
different extension. Commonly the definition includes at least the following
|
||||
items:
|
||||
|
||||
Name|Description|Default or values
|
||||
---|---|--
|
||||
title|Title for the resource|Document name
|
||||
uid|A unique identifier, commonly a GUID|Auto generated
|
||||
documentName|A name of the document such as the file name
|
||||
issueDate|Issued date - When the document was made available. The date can include time
|
||||
modified|Modified - When the document was created or modified. This is the data and time
|
||||
size|Size of document - The document size in bytes
|
||||
url|URL - This can be ``file://``, ``https://``, ``ftp://``, etc resource location
|
||||
|
||||
### Resource optional
|
||||
|
||||
The following are some of the optional items in the resource section. See [Resource](resources.md)
|
||||
for more details
|
||||
|
||||
Name|Description|Default or values
|
||||
---|---|--
|
||||
hash|Hash of document - The hash of the document, which can be blank especially for large documents
|
||||
algo|Hash algorithm - Algoroithm used to generate the hash value or sign it
|
||||
description|Description - A more detailed description
|
||||
version|Version - A document version
|
||||
encoding|Encoding
|
||||
contentType|Content Type
|
||||
compression|Compression|None
|
||||
encryption|Encryption|None
|
||||
author|Author
|
||||
attributes|List of attributes for the resource|Record count
|
||||
|
||||
## Simple sample
|
||||
|
||||
A sample of a single resource **martiLQ** document is shown below. The
|
||||
sample can be generated using the GOLANG client program with parameters:
|
||||
|
||||
```
|
||||
-t GEN -m Sample.json -s ./docs/source/martilq.md --title "GEN001" --description "Simple example"
|
||||
```
|
||||
|
||||
```json
|
||||
{
|
||||
"contentType": "application/vnd.martilq.json",
|
||||
"title": "GEN001",
|
||||
"uid": "9a0a7edb-dd81-4fc5-a6cb-c5716eda7b51",
|
||||
"description": "Simple example",
|
||||
"modified": "2021-11-02T22:44:29.6887001+11:00",
|
||||
"publisher": "",
|
||||
"contactPoint": "",
|
||||
"accessLevel": "Confidential",
|
||||
"rights": "Restricted",
|
||||
"tags": null,
|
||||
"license": "",
|
||||
"state": "active",
|
||||
"batch": 1.001,
|
||||
"describedBy": "",
|
||||
"landingPage": "",
|
||||
"theme": "",
|
||||
"resources": [
|
||||
{
|
||||
"title": "martilq.md",
|
||||
"uid": "a88b4e5f-66b7-4003-ac24-831c95d0da07",
|
||||
"documentName": "martilq.md",
|
||||
"issueDate": "2021-11-02T22:44:29.6881663+11:00",
|
||||
"modified": "2021-11-02T07:47:13.9410018+11:00",
|
||||
"expires": "2023-11-02T00:00:00+11:00",
|
||||
"state": "active",
|
||||
"author": "",
|
||||
"length": 3654,
|
||||
"hash": {
|
||||
"algo": "SHA256",
|
||||
"value": "213a6254ddc02423b6c3bb3d977892678258539d37f06410ef18d27c14ffa821",
|
||||
"signed": false
|
||||
},
|
||||
"description": "",
|
||||
"url": "http://localhost/martilq/martilq.md",
|
||||
"version": "",
|
||||
"contentType": "",
|
||||
"encoding": "UTF-8",
|
||||
"compression": "",
|
||||
"encryption": "",
|
||||
"describedBy": "",
|
||||
"attributes": [
|
||||
{
|
||||
"category": "dataset",
|
||||
"name": "records",
|
||||
"function": "count",
|
||||
"comparison": "EQ",
|
||||
"value": "95"
|
||||
}
|
||||
]
|
||||
}
|
||||
],
|
||||
"custom": [
|
||||
{
|
||||
"extension": "software",
|
||||
"softwareName": "MARTILQREFERENCE",
|
||||
"author": "Meerkat@merebox.com",
|
||||
"version": "0.0.1"
|
||||
}
|
||||
]
|
||||
}
|
||||
```
|
||||
|
||||
You can view a more complete sample [samples/json/sample_02.md](samples/json/sample_02.md)
|
||||
which has been generated using a configuration file to supply default values.
|
||||
|
||||
|
|
@ -0,0 +1,51 @@
|
|||
# Load Quality
|
||||
|
||||
The **martiLQ** document allows for the inclusion of load quality
|
||||
metrics. The load quality metrics is intended to be
|
||||
able to be applied universally with common tools. Not
|
||||
all needs are covered with the base definition but can be extended.
|
||||
|
||||
The load quality metrics are in the majority defined in the [attributes](attributes.md)
|
||||
list attached to each resource. Therefore each resource can have different
|
||||
load quality metrics.
|
||||
|
||||
## Defined load quality metrics
|
||||
|
||||
* Sequential batch number - This is a decimal number defined at the **martiLQ** document
|
||||
header and applies to all resources. The integer portion is for new batches and the fraction
|
||||
part can be used for issues with the same data extract. such as requiring resend because
|
||||
a resource was missing.
|
||||
|
||||
* Number of records in the document - This is the number of data primary records not the
|
||||
count of end of lines and is agreed between parties. XML record counts could be based
|
||||
on the number of primary segments under root. JSON records can be counted in a similar way.
|
||||
The headers or trailling records are not counted
|
||||
|
||||
## Addresses deficiencies
|
||||
|
||||
The **martiLQ** objective is to address deficiencies with alternative
|
||||
data load quality approaches such as:
|
||||
|
||||
* magic formats in file names
|
||||
* identifying the number of files
|
||||
* knowing when all files are ready
|
||||
* separate documentation that is unlinked
|
||||
* securing the data
|
||||
* adding footers to the data, requiring custom file handlers
|
||||
|
||||
## Extending load quality metrics
|
||||
|
||||
**martLQ** document is open to extension so that extra
|
||||
load metrics appropriate to the situation can be included.
|
||||
|
||||
### Extension ideas
|
||||
|
||||
The following extensions for load quality can easily be included:
|
||||
|
||||
* Mandatory data in column
|
||||
* Uniqueness of data values
|
||||
* Data values are within defined tolerances
|
||||
* Check for data exclusions
|
||||
|
||||
And all this information is included in the **martiLQ** document
|
||||
allowing for self describing load quality.
|
||||
|
|
@ -0,0 +1,24 @@
|
|||
# References
|
||||
|
||||
The following are references to documents that inspired the creation of **martiLQ**
|
||||
document and associatd framework.
|
||||
|
||||
https://dex.dss.gov.au/sites/default/files/documents/2021-06/data-exchange-protocols-june-2021.pdf
|
||||
|
||||
https://www.imf.org/en/Data
|
||||
https://www.imf.org/-/media/Files/Publications/WEO/WEO-Database/2021/WEOApr2021all.ashx
|
||||
|
||||
SDMX
|
||||
|
||||
LIXI
|
||||
https://www.imf.org/-/media/Files/Publications/WEO/WEO-Database/2021/WEOApr2021all.ashx
|
||||
|
||||
|
||||
https://standards.theodi.org/introduction/types-of-open-standards-for-data/
|
||||
https://developers.google.com/transit/gtfs/
|
||||
|
||||
http://www.popoloproject.com/
|
||||
|
||||
https://datatracker.ietf.org/doc/html/rfc6350#section-6.2.7
|
||||
https://www.w3.org/TR/vocab-dcat/
|
||||
|
||||
|
|
@ -0,0 +1,98 @@
|
|||
# Resources definition
|
||||
|
||||
The resources section defines the files that are grouped
|
||||
together by association. This association is not defined but can
|
||||
include different formats of the same data or a common batch extract
|
||||
such as end of day.
|
||||
|
||||
Some files may expand to multiple files if they are
|
||||
compressed with a utility such as WinZIP or 7ZIP. In the situation
|
||||
where a ZIP file expands to multiple documents, then the expectation is
|
||||
that the ZIP file contains a **martiLQ** document describing its contents.
|
||||
|
||||
The elements in the resource section are:
|
||||
|
||||
* Title
|
||||
* Document name - Commonly being absolute or relative file name.
|
||||
This value could also be an URL address or network path
|
||||
* Issued date - When the document was made available. The date can include time
|
||||
* Modified - When the document was created or modified. This is the data and time
|
||||
* Size of file - The file size in bytes
|
||||
* Hash of file - The hash of the file, which can be blank especially for large files
|
||||
* Hash algorithm
|
||||
* Attributes - List of attributes associated with the document
|
||||
|
||||
The following are optional in the resource section.
|
||||
|
||||
* Identifier
|
||||
* Description
|
||||
* Download URL
|
||||
* Version - File version. The same file could be updated or this might denote the next version
|
||||
of a regular report. For example a daily extract will have the version number incremented
|
||||
every day and provide a new URL. The previous file can be retained.
|
||||
* Content type - if not specified then the consumer will in all likelihood use the file extension / mime type
|
||||
* Expiry Date - The date and time that this file expires and can be removed from the download URL
|
||||
location. This is not the file retention period as might be required for archiving.
|
||||
* Described By - A link to the metadata describing this file data and format
|
||||
* Compression - Type of compression used if any
|
||||
* Encryption - Type of encryption used if any
|
||||
|
||||
|
||||
## Compression
|
||||
|
||||
Files can be compressed using a utility. A single compressed file can contain
|
||||
multiple files. The **martiLQ** definition document applies to the compressed file
|
||||
and not to the contents, which could be multiple files.
|
||||
|
||||
In the case of a compressed files, there should be a **martiLQ** definition document in the
|
||||
compressed file.
|
||||
|
||||
Compression of files always occur before encryption.
|
||||
|
||||
### martiLQ definition for Compressed File
|
||||
|
||||
For a compressed file that is not encrypted, the distribution definition will be:
|
||||
|
||||
* Title - The compressed file title which could be a group name
|
||||
* Document name - Commonly being absolute or relative file name.
|
||||
This value could also be an URL address or network path
|
||||
* Issued date - When the compressed file was made available.
|
||||
* Modified - When the compressed file was created or modified. This is the date and time
|
||||
and is not the modified date of the file in the compressed file.
|
||||
* Size of file - The compressed file size in bytes
|
||||
* Hash of file - The hash of the compressed file, which can be
|
||||
blank especially for large files
|
||||
* Hash algorithm
|
||||
|
||||
The reason for this approach is it allows a generic tool to be deployed to
|
||||
check the validity of the contents without unpacking the received /fetched
|
||||
file. That is you can perform load quality pipeline processing.
|
||||
|
||||
## Encryption
|
||||
|
||||
The encryption of content is always applied after compression not before, if
|
||||
you are not using the compression tool native encryption. WinZIP and 7ZIP
|
||||
provide encryption within the tool execution.
|
||||
|
||||
If the compression is TAR or GZIP then you may consider applying a GPG
|
||||
or other encryption algorithm to the compressed file.
|
||||
|
||||
* Title - The encrypted file title
|
||||
* Document name - Commonly being absolute or relative file name.
|
||||
This value could also be an URL address or network path
|
||||
* Issued date - When the **encrypted** file was made available.
|
||||
* Modified - When the **encrypted** file was created or modified.
|
||||
This is the data and time and is not the modified date of the encrypted file.
|
||||
* Size of file - The **decrypted** file size in bytes
|
||||
* Hash of file - The hash of the **decrypted** file, which can be
|
||||
blank especially for large files
|
||||
* Hash algorithm
|
||||
|
||||
The rational for using the decrypted file attributes is that an ecrypted
|
||||
file is unlikely to be able to be modified without knowing encryption keys.
|
||||
Checking the decrypted fille attributes is a better check.
|
||||
|
||||
The reason for this approach is it allows a generic tool to be deployed to
|
||||
decrypt and check the validity of the received / fetched file without
|
||||
needing to understand the contents. That is you can perform load quality
|
||||
pipeline processing.
|
||||
|
|
@ -0,0 +1,18 @@
|
|||
# What is marti
|
||||
|
||||
The foundation pillar for the **martiLQ** framework is the [martiLQ document](martiLQ.md)
|
||||
that defines the reconciliation and other metadata of the document / file being transferred.
|
||||
|
||||
A definition, while fundamental, benefits from having tools that can create, read and
|
||||
interpret the definition. The **martiLQ** framework is about providing those tools
|
||||
and an ecosystem that can be added to.
|
||||
|
||||
The majority of the effort in creating **martiLQ** is in the tools. There are tools
|
||||
for various programming languages and situations. As many programming languages
|
||||
generate portable programs that can execute on multiple operating systems, the
|
||||
likelihood is that a tools exists for you.
|
||||
|
||||
The source for tools is provided in the Github repository and some have precompiled
|
||||
images.
|
||||
|
||||
See the project source directory for more details.
|
||||
|
|
@ -0,0 +1,22 @@
|
|||
# When would you use martiLQ
|
||||
|
||||
You are likely to start using the **martiLQ** framework when:
|
||||
|
||||
1. you have no existing standard or framework or;
|
||||
2. your existing standards no longer meets you needs or;
|
||||
3. you are starting document exchange with another business or division that uses the framework or;
|
||||
4. there is a benefit to further automating your document processing in regard to audit and reconciliation
|
||||
|
||||
If you already have a standard and it works for you, and you have no upcoming (large)
|
||||
initiative that would benefit from the framework, then stick with what you have. The benefits
|
||||
of the framework are unlikely to weigh in your framework.
|
||||
|
||||
## Read the material
|
||||
|
||||
Please read the material before jumping in and make sure the **martiLQ** framework will
|
||||
provide you with the expected benefits. Implementing the framework and then changing
|
||||
your direction will more than likely add to your support costs.
|
||||
|
||||
**Note**: As the framework is based on open structure and can be processed independently
|
||||
without the tools, you can export the definitions to other tools / future methods.
|
||||
|
||||
|
|
@ -0,0 +1,41 @@
|
|||
Who is likely to use martiLQ
|
||||
============================
|
||||
|
||||
You are likely to find the **martiLQ** framework relevant if you:
|
||||
|
||||
1. Have many document exchanges, such as End of Day batches
|
||||
2. Need to verify or reconcile the documents
|
||||
|
||||
Data exchanges
|
||||
--------------
|
||||
|
||||
If you are creating or receiving many documents or files on a regular basis
|
||||
then you probably have some framework defined. The framework may be as simple as:
|
||||
|
||||
1. The files are placed in given folders that have significance, such as the source or topic
|
||||
2. File names have a naming standard, such as subject domain and date of extract
|
||||
|
||||
Simple framework such as the above have limitations, such as:
|
||||
|
||||
* File names becoming long and need special parsing, with associated testing
|
||||
* Risk of overwriting
|
||||
* New folders need to be created for new sources
|
||||
* Require constant polling, if passive
|
||||
* Lower automation prospects and alignment to DataSecOps
|
||||
* Poor fit to web applications (they tend to be designed for FTP and LAN)
|
||||
|
||||
Framework Sidecar files
|
||||
-----------------------
|
||||
|
||||
The **martiLQ** framework addresses the issues and limitations by using sidecar
|
||||
or shadow files. The [concept of sidecar files](https://en.wikipedia.org/wiki/Sidecar_file) is
|
||||
not new and are commonly found associated to media file processing.
|
||||
|
||||
Sidecar files can also be implemented as ``forks`` and built into the operating system, such as
|
||||
in Mac OS X HFS. The Microsoft NTFS supports Alternate Data Streams to achieve a similar outcome.
|
||||
Unfortunately this information is not transferrable to other systems.
|
||||
|
||||
The proposition is to define a format for the sidecare file and provide common library tools that
|
||||
can be be used on multiple platforms when exchanging documents / files. Multiple documents can be
|
||||
defined in a singel **martiLQ** definition which adds to efficiency and productivity if used
|
||||
for End of Day or similar batches - or even single file transfers.
|
||||
|
|
@ -0,0 +1,37 @@
|
|||
Why use marti
|
||||
=============
|
||||
|
||||
**martiLQ** is a framework for providing a degree of auditability and reconciliation of
|
||||
documents transferred between systems in an organisation and externally. It does not intend
|
||||
define the format or content of the document. It defines controls that can be used to:
|
||||
|
||||
1. verify source
|
||||
2. identify unauthorized alterations (tampering)
|
||||
3. reconcile agreed metrics, such as the number of records
|
||||
4. ensure sequential processing
|
||||
5. describe metadata including format, extract time
|
||||
6. link to further information
|
||||
|
||||
You would use **martiLQ** if any of the controls are a requirement for you.
|
||||
|
||||
Documents
|
||||
---------
|
||||
|
||||
Documents in this context are digital storage objects such as operating system files,
|
||||
cloud storage objects or blobs. The document content can have structure and contains multiple
|
||||
records or it can be unstructered such as PDFs. If you are including PDFs then they are
|
||||
likely to be be supporting documents for your actual data files.
|
||||
|
||||
The **martiLQ** framework is not intended to be used for single record transfers such as
|
||||
in single web transactions. It is for providing controls when moving large amounts of
|
||||
data as one event. This data are commonly referred to as batch extracts and performed
|
||||
at scheduled times such as end of day.
|
||||
|
||||
Security
|
||||
--------
|
||||
|
||||
The framework does not replace your security, inflight encryption or encryption at rest.
|
||||
|
||||
You are encouraged to use TLS or SSH to connect devices and transfer documents. Storage
|
||||
encryption and access controls for your documents is also relevant as part of the bigger
|
||||
picture.
|
||||
Binary file not shown.
|
|
@ -0,0 +1,196 @@
|
|||
package main
|
||||
|
||||
import (
|
||||
"flag"
|
||||
"log"
|
||||
"net/http"
|
||||
"strings"
|
||||
"path/filepath"
|
||||
"io/ioutil"
|
||||
"os"
|
||||
"encoding/json"
|
||||
)
|
||||
|
||||
func main() {
|
||||
port := flag.String("p", "8080", "Http listen port")
|
||||
staticDirectory := flag.String("static", "static", "Static directory content")
|
||||
docsDirectory := flag.String("docs", "", "Document directory content")
|
||||
dataDirectory := flag.String("data", "", "Data directory content")
|
||||
templateDirectory := flag.String("template", "", "Template directory content")
|
||||
trace := flag.Bool("trace", false, "Produce trace logs")
|
||||
flag.Parse()
|
||||
|
||||
if *trace == true {
|
||||
log.Printf("static folder: %s\n", *staticDirectory)
|
||||
log.Printf("data folder: %s\n", *dataDirectory)
|
||||
log.Printf("docs folder: %s\n", *docsDirectory)
|
||||
log.Printf("template folder: %s\n", *templateDirectory)
|
||||
}
|
||||
|
||||
http.HandleFunc("/data/", func( res http.ResponseWriter, req *http.Request ) {
|
||||
safePath := ValidatePath(req.URL.Path[1:])
|
||||
if (*dataDirectory != "") {
|
||||
safePath = filepath.FromSlash(filepath.Join(*dataDirectory, strings.Replace(safePath, "data/", "", 1)))
|
||||
}
|
||||
if *trace == true {
|
||||
log.Printf("resolved data folder: %s\n", safePath)
|
||||
}
|
||||
http.ServeFile(res, req, safePath)
|
||||
})
|
||||
|
||||
http.HandleFunc("/template/", func( res http.ResponseWriter, req *http.Request ) {
|
||||
safePath := ValidatePath(req.URL.Path[1:])
|
||||
if (*templateDirectory != "") {
|
||||
safePath = filepath.FromSlash(filepath.Join(*templateDirectory, strings.Replace(safePath, "template/", "", 1)))
|
||||
}
|
||||
if *trace == true {
|
||||
log.Printf("resolved template folder: %s\n", safePath)
|
||||
}
|
||||
http.ServeFile(res, req, safePath)
|
||||
})
|
||||
|
||||
http.HandleFunc("/docs/", func( res http.ResponseWriter, req *http.Request ) {
|
||||
localPath := ""
|
||||
if (*docsDirectory == "") {
|
||||
temp := "../../.."
|
||||
docsDirectory = &temp
|
||||
localPath = ValidatePath(filepath.FromSlash(*docsDirectory+req.URL.Path))
|
||||
} else {
|
||||
localPath = ValidatePath(filepath.FromSlash(*docsDirectory+strings.Replace(req.URL.Path, "docs/", "", 1)))
|
||||
}
|
||||
if *trace == true {
|
||||
log.Printf("resolved docs folder: \"%s\"", localPath)
|
||||
}
|
||||
f, err := os.Open(localPath)
|
||||
if err != nil {
|
||||
log.Printf("fetch docs error: \"%s\" with %s", localPath, err)
|
||||
http.NotFound(res, req)
|
||||
} else {
|
||||
s, err := f.Stat()
|
||||
if err != nil || s.IsDir() {
|
||||
log.Printf("fetch docs stat error: \"%s\"", localPath)
|
||||
http.NotFound(res, req)
|
||||
//http.ServeFile(res, req, filepath.FromSlash(*staticDirectory + "/404.html"))
|
||||
} else {
|
||||
http.ServeFile(res, req, localPath)
|
||||
}
|
||||
}
|
||||
})
|
||||
|
||||
|
||||
http.HandleFunc("/api/", apiHandler)
|
||||
http.HandleFunc("/api/view", apiHandlerView)
|
||||
|
||||
fileServer := http.FileServer(FileSystem{http.Dir(*staticDirectory)})
|
||||
http.Handle("/", fileServer)
|
||||
|
||||
log.Printf("Serving on HTTP port: %s\n", *port)
|
||||
log.Fatal(http.ListenAndServe(":"+*port, nil))
|
||||
}
|
||||
|
||||
|
||||
|
||||
type FileSystem struct {
|
||||
fs http.FileSystem
|
||||
}
|
||||
|
||||
func (fs FileSystem) Open(path string) (http.File, error) {
|
||||
f, err := fs.fs.Open(path)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
s, err := f.Stat()
|
||||
if s.IsDir() {
|
||||
index := strings.TrimSuffix(path, "/") + "/index.html"
|
||||
if _, err := fs.fs.Open(index); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
}
|
||||
|
||||
return f, nil
|
||||
}
|
||||
|
||||
func ValidatePath(path string) string {
|
||||
|
||||
safePath := path
|
||||
|
||||
return safePath
|
||||
}
|
||||
|
||||
func apiHandler(res http.ResponseWriter, req *http.Request) {
|
||||
|
||||
apiPath := req.URL.Path
|
||||
log.Printf("fetch api: \"%s\"", apiPath)
|
||||
|
||||
res.Write([]byte("<h1>Welcome to my web server!</h1>"))
|
||||
}
|
||||
|
||||
|
||||
type oTemplate struct {
|
||||
Extension string `json:"extension"`
|
||||
Renderer string `json:"renderer"`
|
||||
Url string `json:"url"`
|
||||
}
|
||||
|
||||
type Definition struct {
|
||||
FileName string `json:"fileName"`
|
||||
Describe string `json:"describe"`
|
||||
}
|
||||
|
||||
type DirectoryList struct {
|
||||
Custom []oTemplate `json:"custom"`
|
||||
Files []Definition `json:"files"`
|
||||
}
|
||||
|
||||
func apiHandlerView(res http.ResponseWriter, req *http.Request) {
|
||||
|
||||
dataFolder := "data/"
|
||||
|
||||
if req.Method == "GET" {
|
||||
res.Header().Set("contentType", "application/json")
|
||||
|
||||
files, err := ioutil.ReadDir(dataFolder)
|
||||
if err != nil {
|
||||
log.Fatal(err)
|
||||
}
|
||||
|
||||
template := oTemplate{Extension: "template", Renderer: "MARTILQREFERENCE:Mustache", Url: "template/martilq_view.must"}
|
||||
fileList := []Definition{}
|
||||
list := DirectoryList{}
|
||||
list.Custom = append(list.Custom, template)
|
||||
|
||||
for _, file := range files {
|
||||
if !file.IsDir() {
|
||||
if filepath.Ext(file.Name()) == ".json" {
|
||||
describe := ""
|
||||
// Fetch the description
|
||||
data, err := ioutil.ReadFile(filepath.Join(dataFolder,file.Name()))
|
||||
if err != nil {
|
||||
log.Fatal("error with file read ")
|
||||
} else {
|
||||
|
||||
var unknown map[string]interface{}
|
||||
err = json.Unmarshal(data, &unknown)
|
||||
if err != nil {
|
||||
log.Fatal("error with json read ")
|
||||
} else {
|
||||
describe = unknown["title"].(string)
|
||||
}
|
||||
}
|
||||
|
||||
def := Definition{FileName: file.Name(), Describe: describe}
|
||||
fileList = append(fileList, def)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
list.Files = fileList
|
||||
|
||||
content, _ := json.Marshal(list)
|
||||
res.Write([]byte(content))
|
||||
} else {
|
||||
http.Error(res, "Only GET requests are allowed!", http.StatusMethodNotAllowed)
|
||||
}
|
||||
|
||||
}
|
||||
|
Before Width: | Height: | Size: 1007 B After Width: | Height: | Size: 1007 B |
|
Before Width: | Height: | Size: 2.0 KiB After Width: | Height: | Size: 2.0 KiB |
|
Before Width: | Height: | Size: 5.2 KiB After Width: | Height: | Size: 5.2 KiB |
|
|
@ -83,6 +83,8 @@
|
|||
|
||||
<script src="assets/dist/js/bootstrap.bundle.min.js"></script>
|
||||
<script src="https://cdn.jsdelivr.net/npm/feather-icons@4.28.0/dist/feather.min.js" integrity="sha384-uO3SXW5IuS1ZpFPKugNNWqTZRRglnUJK6UAZ/gxOX80nxEkN9NcGZTftn6RzhGWE" crossorigin="anonymous"></script><script src="https://cdn.jsdelivr.net/npm/chart.js@2.9.4/dist/Chart.min.js" integrity="sha384-zNy6FEbO50N+Cg5wap8IKA4M/ZnLJgzc6w2NqACZaK0u0FXfOWRRJOnQtpZun8ha" crossorigin="anonymous"></script>
|
||||
<script src="https://cdnjs.cloudflare.com/ajax/libs/mustache.js/2.3.0/mustache.min.js"></script>
|
||||
<script src="js/martilq_app.js"></script>
|
||||
|
||||
</body>
|
||||
</html>
|
||||
|
|
@ -2,10 +2,12 @@
|
|||
"use strict";
|
||||
|
||||
const output = document.querySelector(".output");
|
||||
var localJsonFile = "data/marti_test_asic.json";
|
||||
var localJsonFile = "";
|
||||
|
||||
var btn = document.getElementById("loadBtn");
|
||||
btn.onclick = dataLoadFunction;
|
||||
if (btn) {
|
||||
btn.onclick = dataLoadFunction;
|
||||
}
|
||||
|
||||
function dataLoadFunction() {
|
||||
var loadDef = document.getElementById("loaddefinition")
|
||||
|
|
@ -31,7 +33,9 @@ window.addEventListener("DOMContentLoaded", () => {
|
|||
fetchData("data/"+ loadDef);
|
||||
}
|
||||
var ld = document.getElementById("loaddefinition")
|
||||
ld.value = loadDef
|
||||
if (ld) {
|
||||
ld.value = loadDef
|
||||
}
|
||||
} else {
|
||||
output.innerHTML = "Please supply a MartiLQ definition to load, such as \"<a href=\"?martilq=martilq_asic.json\">martilq_asic.json</a>\"";
|
||||
}
|
||||
|
|
@ -44,13 +48,15 @@ function fetchData(dataFile) {
|
|||
.then((response) => response.json())
|
||||
.then((data) => {
|
||||
|
||||
var template = "data/martilq_def.must";
|
||||
var template = "template/martilq_default.must";
|
||||
jdata["item"] = data;
|
||||
jdata["describe"] = data.description.replace(/\r\n/g, "<br>");
|
||||
if (data.description) {
|
||||
jdata["describe"] = data.description.replace(/\r\n/g, "<br>");
|
||||
}
|
||||
|
||||
if (data["custom"]) {
|
||||
data.custom.forEach((el) => {
|
||||
if (el.extension == "template" && el.renderer == "MartiReference:Mustache") {
|
||||
if (el.extension == "template" && el.renderer == "MARTILQREFERENCE:Mustache") {
|
||||
template = el.url;
|
||||
}
|
||||
});
|
||||
|
|
@ -0,0 +1,58 @@
|
|||
<!doctype html>
|
||||
<html lang="en">
|
||||
<head>
|
||||
<meta charset="utf-8">
|
||||
<meta name="viewport" content="width=device-width, initial-scale=1">
|
||||
<meta name="description" content="">
|
||||
<meta name="author" content="Mark Otto, Jacob Thornton, and Bootstrap contributors">
|
||||
<meta name="generator" content="Hugo 0.88.1">
|
||||
<title>MartiLQ Definition</title>
|
||||
|
||||
<!-- Bootstrap core CSS -->
|
||||
<link href="assets/dist/css/bootstrap.min.css" rel="stylesheet">
|
||||
|
||||
<style>
|
||||
.bd-placeholder-img {
|
||||
font-size: 1.125rem;
|
||||
text-anchor: middle;
|
||||
-webkit-user-select: none;
|
||||
-moz-user-select: none;
|
||||
user-select: none;
|
||||
}
|
||||
|
||||
@media (min-width: 768px) {
|
||||
.bd-placeholder-img-lg {
|
||||
font-size: 3.5rem;
|
||||
}
|
||||
}
|
||||
</style>
|
||||
<link href="css/dashboard.css" rel="stylesheet">
|
||||
</head>
|
||||
<body>
|
||||
|
||||
<header class="navbar navbar-dark sticky-top bg-dark flex-md-nowrap p-0 shadow">
|
||||
<a class="navbar-brand col-md-3 col-lg-2 me-0 px-3" href="https://github.com/meerkat-manor/marti">MartiLQ</a>
|
||||
</header>
|
||||
|
||||
<div class="container-fluid">
|
||||
<div class="row">
|
||||
|
||||
<main class="col-md-9 ms-sm-auto col-lg-10 px-md-4">
|
||||
<div class="table-responsive">
|
||||
<div class="output"></div>
|
||||
</div>
|
||||
</main>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<footer class="bg-dark flex-md-nowrap p-0 shadow">
|
||||
|
||||
</footer>
|
||||
|
||||
<script src="assets/dist/js/bootstrap.bundle.min.js"></script>
|
||||
<script src="https://cdn.jsdelivr.net/npm/feather-icons@4.28.0/dist/feather.min.js" integrity="sha384-uO3SXW5IuS1ZpFPKugNNWqTZRRglnUJK6UAZ/gxOX80nxEkN9NcGZTftn6RzhGWE" crossorigin="anonymous"></script><script src="https://cdn.jsdelivr.net/npm/chart.js@2.9.4/dist/Chart.min.js" integrity="sha384-zNy6FEbO50N+Cg5wap8IKA4M/ZnLJgzc6w2NqACZaK0u0FXfOWRRJOnQtpZun8ha" crossorigin="anonymous"></script>
|
||||
<script src="https://cdnjs.cloudflare.com/ajax/libs/mustache.js/2.3.0/mustache.min.js"></script>
|
||||
<script src="js/martilq_app.js"></script>
|
||||
|
||||
</body>
|
||||
</html>
|
||||
|
|
@ -0,0 +1,79 @@
|
|||
<h2>Metadata definition</h2>
|
||||
{{#item}}
|
||||
<br>
|
||||
<h3>File details</h3>
|
||||
|
||||
<table class="table table-striped table-sm">
|
||||
<tr><th>Description</th><td>{{describe}}</td></tr>
|
||||
<tr><th>Encoding</th><td>{{encoding}}</td></tr>
|
||||
<tr><th>Format</th><td>{{format}}</td></tr>
|
||||
<tr><th>Multi record</th><td>{{multiRecord}}</td></tr>
|
||||
<tr><th>Line Endings</th><td>{{lineEndings}}</td></tr>
|
||||
<tr><th>Header</th><td>{{header}}</td></tr>
|
||||
<tr><th>Footer</th><td>{{footer}}</td></tr>
|
||||
<tr><th>Delimiter</th><td>{{delimiter}}</td></tr>
|
||||
<tr><th>Quote</th><td>{{quote}}</td></tr>
|
||||
<tr><th>Escape</th><td>{{escape}}</td></tr>
|
||||
</table>
|
||||
|
||||
{{#recordFormat}}
|
||||
<br>
|
||||
<h3>{{formatName}} Columns</h3>
|
||||
<table class="table table-striped table-sm">
|
||||
<thead>
|
||||
<th>Name</th><th>Description</th><th>Type</th><th>Length</th><th>Nullable</th><th>Default</th><th>Dataset</th>
|
||||
<thead>
|
||||
<tbody>
|
||||
{{#columns}}
|
||||
<tr>
|
||||
<td>{{columnName}}</td>
|
||||
<td>{{description}}</td>
|
||||
<td>{{type}}</td>
|
||||
<td>{{length}}</td>
|
||||
<td>{{nullable}}</td>
|
||||
<td>{{default}}</td>
|
||||
<td>{{dataset}}</td>
|
||||
</tr>
|
||||
<tr>
|
||||
<td></td>
|
||||
<td>Format: {{formatting}}</td>
|
||||
<td colspan="2">L Label: {{labels.leftLabel}}</td>
|
||||
<td colspan="2">R Label: {{labels.rightLabel}}</td>
|
||||
<td colspan="2">Heading: {{labels.heading}}</td>
|
||||
</tr>
|
||||
{{/columns}}
|
||||
</tbody>
|
||||
</table>
|
||||
{{/recordFormat}}
|
||||
|
||||
<br>
|
||||
<h3>Index</h3>
|
||||
<table class="table table-striped table-sm">
|
||||
<thead>
|
||||
<th>Name</th><th>Primary</th><th>Unique</th><th>Columns</th>
|
||||
<thead>
|
||||
<tbody>
|
||||
{{#indexes}}
|
||||
<tr>
|
||||
<td>{{indexName}}</td>
|
||||
<td>{{primary}}</td>
|
||||
<td>{{unique}}</td>
|
||||
<td>
|
||||
{{#columns}}
|
||||
{{name}},
|
||||
{{/columns}}
|
||||
</td>
|
||||
</tr>
|
||||
{{/indexes}}
|
||||
</tbody>
|
||||
</table>
|
||||
|
||||
<br><hr>
|
||||
<h3>Sample</h3>
|
||||
<pre>
|
||||
{{#sample}}
|
||||
{{record}}
|
||||
{{/sample}}
|
||||
</pre>
|
||||
{{/item}}
|
||||
<p font="small"><a href="/template/details/martilq_cols_csv.must">Default columns CSV template</a>. Version: 202111B</p>
|
||||
|
|
@ -0,0 +1,75 @@
|
|||
<h2>Metadata definition</h2>
|
||||
{{#item}}
|
||||
<br>
|
||||
<h3>File details</h3>
|
||||
|
||||
<table class="table table-striped table-sm">
|
||||
<tr><th>Description</th><td>{{describe}}</td></tr>
|
||||
<tr><th>Encoding</th><td>{{encoding}}</td></tr>
|
||||
<tr><th>Format</th><td>{{format}}</td></tr>
|
||||
<tr><th>Multi record</th><td>{{multiRecord}}</td></tr>
|
||||
<tr><th>Line Endings</th><td>{{lineEndings}}</td></tr>
|
||||
<tr><th>Header</th><td>{{header}}</td></tr>
|
||||
<tr><th>Footer</th><td>{{footer}}</td></tr>
|
||||
</table>
|
||||
|
||||
{{#recordFormat}}
|
||||
<br>
|
||||
<h3>{{formatName}} Columns</h3>
|
||||
<table class="table table-striped table-sm">
|
||||
<thead>
|
||||
<th>Name</th><th>Description</th><th>Type</th><th>Starting</th><th>Ending</th><th>Dataset</th>
|
||||
<thead>
|
||||
<tbody>
|
||||
{{#columns}}
|
||||
<tr>
|
||||
<td>{{columnName}}</td>
|
||||
<td>{{description}}</td>
|
||||
<td>{{type}}</td>
|
||||
<td>{{starting}}</td>
|
||||
<td>{{ending}}</td>
|
||||
<td>{{dataset}}</td>
|
||||
</tr>
|
||||
<tr>
|
||||
<td></td>
|
||||
<td>Format: {{formatting}}</td>
|
||||
<td colspan="2">L Label: {{labels.leftLabel}}</td>
|
||||
<td colspan="2">R Label: {{labels.rightLabel}}</td>
|
||||
<td colspan="2">Heading: {{labels.heading}}</td>
|
||||
</tr>
|
||||
{{/columns}}
|
||||
</tbody>
|
||||
</table>
|
||||
{{/recordFormat}}
|
||||
|
||||
<br>
|
||||
<h3>Index</h3>
|
||||
<table class="table table-striped table-sm">
|
||||
<thead>
|
||||
<th>Name</th><th>Primary</th><th>Unique</th><th>Columns</th>
|
||||
<thead>
|
||||
<tbody>
|
||||
{{#indexes}}
|
||||
<tr>
|
||||
<td>{{indexName}}</td>
|
||||
<td>{{primary}}</td>
|
||||
<td>{{unique}}</td>
|
||||
<td>
|
||||
{{#columns}}
|
||||
{{name}},
|
||||
{{/columns}}
|
||||
</td>
|
||||
</tr>
|
||||
{{/indexes}}
|
||||
</tbody>
|
||||
</table>
|
||||
|
||||
<br><hr>
|
||||
<h3>Sample</h3>
|
||||
<pre>
|
||||
{{#sample}}
|
||||
{{record}}
|
||||
{{/sample}}
|
||||
</pre>
|
||||
{{/item}}
|
||||
<p font="small"><a href="/template/details/martilq_cols_fco.must">Default columns FCP template</a>. Version: 202111B</p>
|
||||
|
|
@ -21,7 +21,7 @@
|
|||
<br>
|
||||
<table class="table table-striped table-sm">
|
||||
<thead>
|
||||
<th>Title</th><th>Document</th><th>Size</th><th>Issued</th><th>Modified</th><th>Format</th><th>State</th><th>Version</th>
|
||||
<th>Title</th><th>Document</th><th>Size</th><th>Issued</th><th>Modified</th><th>Format</th><th>State</th><th>Version</th><th>Structure</th>
|
||||
<thead>
|
||||
<tbody>
|
||||
{{#resources}}
|
||||
|
|
@ -34,8 +34,10 @@
|
|||
<td>{{format}}</td>
|
||||
<td>{{state}}</td>
|
||||
<td>{{version}}</td>
|
||||
<td><a href="{{structure}}">Structure</a></td>
|
||||
</tr>
|
||||
{{/resources}}
|
||||
</tbody>
|
||||
</table>
|
||||
{{/item}}
|
||||
<p font="small"><a href="/template/martilq_ckan.must">CKAN template</a>. Version: 202111B</p>
|
||||
|
|
@ -16,7 +16,7 @@
|
|||
<br>
|
||||
<table class="table table-striped table-sm">
|
||||
<thead>
|
||||
<th>Title</th><th>Document</th><th>Size</th><th>Issued</th><th>Modified</th><th>Expires</th><th>State</th><th>Version</th>
|
||||
<th>Title</th><th>Document</th><th>Size</th><th>Issued</th><th>Modified</th><th>Expires</th><th>State</th><th>Version</th><th>Structure</th>
|
||||
<thead>
|
||||
<tbody>
|
||||
{{#resources}}
|
||||
|
|
@ -29,8 +29,10 @@
|
|||
<td>{{expires}}</td>
|
||||
<td>{{state}}</td>
|
||||
<td>{{version}}</td>
|
||||
<td><a href="{{structure}}">Structure</a></td>
|
||||
</tr>
|
||||
{{/resources}}
|
||||
</tbody>
|
||||
</table>
|
||||
{{/item}}
|
||||
<p font="small"><a href="/template/martilq_custom.must">Custom template</a>. Version: 202111B</p>
|
||||
|
|
@ -0,0 +1,41 @@
|
|||
|
||||
<br>
|
||||
<table class="table table-striped table-sm">
|
||||
<tr><th>Title</th><td>{{item.title}}</td></tr>
|
||||
<tr><th>UID</th><td>{{item.uid}}</td></tr>
|
||||
<tr><th>Description</th><td>{{{describe}}}</td></tr>
|
||||
<tr><th>State</th><td>{{{item.state}}}</td></tr>
|
||||
<tr><th>Issued</th><td>{{item.issued}}</td></tr>
|
||||
<tr><th>Modified</th><td>{{item.modified}}</td></tr>
|
||||
<tr><th>Expires</th><td>{{item.expires}}</td></tr>
|
||||
<tr><th>Tags</th><td>{{item.tags}}</td></tr>
|
||||
<tr><th>Access Level</th><td>{{item.accessLevel}}</td></tr>
|
||||
<tr><th>Rights</th><td>{{item.rights}}</td></tr>
|
||||
<tr><th>License</th><td>{{item.license}}</td></tr>
|
||||
<tr><th>Contact</th><td>{{item.contactPoint}}</td></tr>
|
||||
</table>
|
||||
|
||||
{{#item}}
|
||||
<br>
|
||||
<table class="table table-striped table-sm">
|
||||
<thead>
|
||||
<th>Title</th><th>Document</th><th>Size</th><th>Issued</th><th>Modified</th><th>Format</th><th>State</th><th>Version</th><th>Structure</th>
|
||||
<thead>
|
||||
<tbody>
|
||||
{{#resources}}
|
||||
<tr>
|
||||
<td>{{title}}</td>
|
||||
<td><a href="{{url}}">{{documentName}}</a></td>
|
||||
<td>{{length}}</td>
|
||||
<td>{{issuedDate}}</td>
|
||||
<td>{{modified}}</td>
|
||||
<td>{{contentType}}</td>
|
||||
<td>{{state}}</td>
|
||||
<td>{{version}}</td>
|
||||
<td><a href="{{structure}}">Structure</a></td>
|
||||
</tr>
|
||||
{{/resources}}
|
||||
</tbody>
|
||||
</table>
|
||||
{{/item}}
|
||||
<p font="small"><a href="/template/martilq_default.must">Default template</a>. Version: 202111B</p>
|
||||
|
|
@ -12,6 +12,7 @@
|
|||
<tr><th>Access Level</th><td>{{item.accessLevel}}</td></tr>
|
||||
<tr><th>Rights</th><td>{{item.rights}}</td></tr>
|
||||
<tr><th>License</th><td>{{item.license}}</td></tr>
|
||||
<tr><th>Contact</th><td>{{item.contactPoint}}</td></tr>
|
||||
</table>
|
||||
|
||||
{{#item}}
|
||||
|
|
@ -28,7 +29,7 @@
|
|||
<td>{{length}}</td>
|
||||
<td>{{issuedDate}}</td>
|
||||
<td>{{modified}}</td>
|
||||
<td>{{format}}</td>
|
||||
<td>{{contentType}}</td>
|
||||
<td>{{state}}</td>
|
||||
<td>{{version}}</td>
|
||||
</tr>
|
||||
|
|
@ -36,3 +37,4 @@
|
|||
</tbody>
|
||||
</table>
|
||||
{{/item}}
|
||||
<p font="small"><a href="/template/martilq_no_struct.must">No structure template</a>. Version: 202111B</p>
|
||||
|
|
@ -0,0 +1,20 @@
|
|||
<h2>MartiLQ definition</h2>
|
||||
<p>The following <string>martiLQ</string> definitions are provided as samples</p>
|
||||
<p>The <a href="https://github.com/meerkat-manor/marti">server code</a> for this is available and the code can be changed to reflect your actual or sample definitions</p>
|
||||
{{#item}}
|
||||
<br>
|
||||
<table class="table table-striped table-sm">
|
||||
<thead>
|
||||
<th>Definition</th><th>Document</th>
|
||||
<thead>
|
||||
<tbody>
|
||||
{{#files}}
|
||||
<tr>
|
||||
<td><a href="browse.html?martilq={{fileName}}">{{fileName}}</a></td>
|
||||
<td>{{describe}}</td>
|
||||
</tr>
|
||||
{{/files}}
|
||||
</tbody>
|
||||
</table>
|
||||
{{/item}}
|
||||
<p font="small"><a href="/template/martilq_view.must">View template</a>. Version: 202111B</p>
|
||||
|
|
@ -21,8 +21,9 @@ public class Resource {
|
|||
|
||||
public String description;
|
||||
public String url;
|
||||
public String structure;
|
||||
public String version;
|
||||
public String format;
|
||||
public String content_type;
|
||||
public String compression;
|
||||
public String encryption;
|
||||
|
||||
|
|
|
|||
|
|
@ -1,15 +1,10 @@
|
|||
|
||||
#. ..\..\..\source\powershell\MartiLQUtilities.ps1
|
||||
|
||||
. .\source\powershell\MartiLQUtilities.ps1
|
||||
. .\source\powershell\MartiLQConfiguration.ps1
|
||||
. .\source\powershell\MartiLQResource.ps1
|
||||
. .\source\powershell\MartiLQAttribute.ps1
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
function New-MartiDefinition
|
||||
{
|
||||
|
||||
|
|
@ -20,22 +15,31 @@ function New-MartiDefinition
|
|||
version = "$script:SoftwareVersion"
|
||||
}
|
||||
|
||||
$oTemplate = [PSCustomObject]@{
|
||||
extension = "template"
|
||||
renderer = "MARTILQREFERENCE:Mustache"
|
||||
url = ""
|
||||
}
|
||||
|
||||
$publisher = [System.Security.Principal.WindowsIdentity]::GetCurrent().Name
|
||||
|
||||
[System.Collections.ArrayList]$lcustom = @()
|
||||
$lcustom += $oSoftware
|
||||
$lcustom += $oTemplate
|
||||
|
||||
[System.Collections.ArrayList]$lresource = @()
|
||||
|
||||
$expires = (Get-Date).AddYears(7)
|
||||
|
||||
$oMarti = [PSCustomObject]@{
|
||||
"content-type" = "application/vnd.martilq.json"
|
||||
contentType = "application/vnd.martilq.json"
|
||||
title = ""
|
||||
uid = (New-Guid).ToString()
|
||||
|
||||
description = ""
|
||||
issued = Get-Date -f "yyyy-MM-ddTHH:mm:ss"
|
||||
modified = Get-Date -f "yyyy-MM-ddTHH:mm:ss"
|
||||
expires = ""
|
||||
expires = $expires -f "yyyy-MM-ddTHH:mm:ss"
|
||||
tags = @( "document", "marti")
|
||||
publisher = $publisher
|
||||
contactPoint = ""
|
||||
|
|
@ -218,13 +222,15 @@ Param(
|
|||
}
|
||||
|
||||
$hash = New-MartiHash -Algorithm "SHA256" -FilePath "" -Value $_.hash
|
||||
$expires = (Get-Date).AddYears(7)
|
||||
|
||||
$oResource = [PSCustomObject]@{
|
||||
title = $_.name
|
||||
uid = $_.id
|
||||
documentName = $name
|
||||
issuedDate = $_.created
|
||||
modified = $_.last_modified
|
||||
issuedDate = $_.created.ToString("yyyy-MM-ddTHH:mm:ss")
|
||||
modified = $_.last_modified.ToString("yyyy-MM-ddTHH:mm:ss")
|
||||
expires = $expires.Tostring("yyyy-MM-ddTHH:mm:ss")
|
||||
state = $_.state
|
||||
author = $oCkan.result.author
|
||||
length = $_.size
|
||||
|
|
@ -232,10 +238,11 @@ Param(
|
|||
|
||||
description = $_.description
|
||||
url = $_.url
|
||||
structure = $null
|
||||
version = $version
|
||||
format = $_.format
|
||||
compression = ""
|
||||
encryption = ""
|
||||
contentType = Get-MimeType(("."+$_.format))
|
||||
compression = $null
|
||||
encryption = $null
|
||||
}
|
||||
|
||||
$lresource += $oResource
|
||||
|
|
|
|||
|
|
@ -1,4 +1,44 @@
|
|||
|
||||
function Get-MimeType {
|
||||
Param(
|
||||
[Parameter(Mandatory)][String] $Extension
|
||||
)
|
||||
|
||||
$mimeType = "application/unknown";
|
||||
if ( $null -ne $Extension )
|
||||
{
|
||||
Switch ($Extension)
|
||||
{
|
||||
".json" { $mimetype = "application/json" ; break }
|
||||
".md" { $mimetype = "text/markdown" ; break }
|
||||
".yml" { $mimetype = "text/yaml" ; break }
|
||||
".rst" { $mimetype = "text/x-rst" ; break }
|
||||
".7z" { $mimetype = "application/x-7z-compressed" ; break }
|
||||
".mti" { $mimetype = "application/vnd.martilq.json" ; break }
|
||||
".ttf" { $mimetype = $null ; break }
|
||||
".eot" { $mimetype = $null ; break }
|
||||
".woff" { $mimetype = $null ; break }
|
||||
".woff2" { $mimetype = $null ; break }
|
||||
".csv" { $mimetype = "text/csv" ; break }
|
||||
".tsv" { $mimetype = "text/csv" ; break }
|
||||
Default {
|
||||
$drive = Get-PSDrive HKCR -ErrorAction SilentlyContinue;
|
||||
if ( $null -eq $drive )
|
||||
{
|
||||
$drive = New-PSDrive -Name HKCR -PSProvider Registry -Root HKEY_CLASSES_ROOT
|
||||
}
|
||||
$ext = Get-ItemProperty HKCR:$Extension -ErrorAction SilentlyContinue;
|
||||
if ( $null -ne $ext) {
|
||||
$mimeType = $ext."Content Type";
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return $mimeType
|
||||
|
||||
}
|
||||
|
||||
|
||||
function New-MartiResource {
|
||||
Param(
|
||||
|
|
@ -45,14 +85,15 @@ Param(
|
|||
modified = $item.LastWriteTime.ToString("yyyy-MM-ddTHH:mm:ss")
|
||||
expires = $expires.ToString("yyyy-MM-ddTHH:mm:ss")
|
||||
state = "active"
|
||||
author = ""
|
||||
author = $null
|
||||
length = $item.Length
|
||||
hash = $hash
|
||||
|
||||
description = ""
|
||||
url = ""
|
||||
description = $null
|
||||
url = $null
|
||||
structure = $null
|
||||
version = $version
|
||||
format = $item.Extension.Substring(1)
|
||||
contentType = Get-MimeType($item.Extension)
|
||||
compression = $null
|
||||
encryption = $null
|
||||
|
||||
|
|
@ -531,7 +572,7 @@ function Compare-MartiResource {
|
|||
$formatProcessed = $false
|
||||
[System.Collections.ArrayList]$lerror = @()
|
||||
|
||||
if ($Resource.format -eq "CSV") {
|
||||
if ($Resource.contentType -eq "text/csv") {
|
||||
$formatProcessed = $true
|
||||
|
||||
$data = $inputData | ConvertFrom-Csv -Delim ','
|
||||
|
|
@ -573,7 +614,7 @@ function Compare-MartiResource {
|
|||
}
|
||||
|
||||
|
||||
if ($Resource.format -eq "JSON") {
|
||||
if ($Resource.contentType -eq "application/json") {
|
||||
$formatProcessed = $true
|
||||
|
||||
$data = $inputData | ConvertFrom-Json
|
||||
|
|
|
|||
|
|
@ -30,6 +30,12 @@ class martiLQ:
|
|||
"version": "0.0.1"
|
||||
}
|
||||
|
||||
_oTemplate = {
|
||||
"extension": "template",
|
||||
"renderer": "MARTILQREFERENCE:Mustache",
|
||||
"url": ""
|
||||
}
|
||||
|
||||
|
||||
_MartiErrorId = ""
|
||||
_oMartiDefinition = None
|
||||
|
|
@ -99,16 +105,21 @@ class martiLQ:
|
|||
publisher = getpass.getuser()
|
||||
|
||||
lcustom = []
|
||||
self._oSoftware["softwareName"] = self.GetSoftwareName()
|
||||
lcustom.append(self._oSoftware)
|
||||
self._oTemplate["renderer"] = self.GetSoftwareName() + ":Mustache"
|
||||
self._oTemplate["url"] = "template/martilq_ckan.must"
|
||||
lcustom.append(self._oTemplate)
|
||||
|
||||
lresource = []
|
||||
|
||||
self._oMartiDefinition = {
|
||||
"content-type": "application/vnd.martilq.json",
|
||||
"contentType": "application/vnd.martilq.json",
|
||||
"title": "",
|
||||
"uid": str(uuid.uuid4()),
|
||||
|
||||
"description": "",
|
||||
"issued": dateToday,
|
||||
"modified": dateToday,
|
||||
"publisher": publisher,
|
||||
"contactPoint": self._oConfiguration.GetConfig("contactPoint"),
|
||||
|
|
|
|||
|
|
@ -109,8 +109,9 @@ class mResource:
|
|||
|
||||
"description": "",
|
||||
"url": self._oConfiguration.GetConfig("urlPrefix"),
|
||||
"structure": "",
|
||||
"version": self._oConfiguration.GetConfig("version"),
|
||||
"content-type": self.GetContentType(SourcePath),
|
||||
"contentType": self.GetContentType(SourcePath),
|
||||
"encoding": self._oConfiguration.GetConfig("encoding"),
|
||||
"compression": self._oConfiguration.GetConfig("compression"),
|
||||
"encryption": self._oConfiguration.GetConfig("encryption"),
|
||||
|
|
@ -136,6 +137,36 @@ class mResource:
|
|||
return oResource
|
||||
|
||||
|
||||
def SetAttributeValueString(oResource, Category, Key, Function, Value, Comparison="EQ"):
|
||||
|
||||
for item in oResource["attributes"]:
|
||||
|
||||
if item["category"] == Category and item["name"] == Key and item["function"] == Function:
|
||||
if item["comparison"] == "NA" or item["comparison"] == Comparison:
|
||||
item["comparison"] = Comparison
|
||||
item["value"] = Value
|
||||
return
|
||||
|
||||
# Add the attribute
|
||||
oAttribute = {
|
||||
"category": Category,
|
||||
"name": Key,
|
||||
"function": Function,
|
||||
"comparison": Comparison,
|
||||
"value": Value
|
||||
}
|
||||
|
||||
oResource["attributes"].append(oAttribute)
|
||||
|
||||
return
|
||||
|
||||
|
||||
def SetAttributeValueNumber(oResource, Category, Key, Function, Value, Comparison="EQ"):
|
||||
|
||||
mResource.SetAttributeValueString(oResource, Category, Key, Function, Value, Comparison)
|
||||
|
||||
return
|
||||
|
||||
|
||||
def NewMartiLQHash(self, Algorithm, FilePath, Value="", Sign=""):
|
||||
|
||||
|
|
|
|||
|
|
@ -0,0 +1,27 @@
|
|||
|
||||
|
||||
To execute the PowerShell scripts, please invoke from the root Marti directory and not from
|
||||
with the current directory set to ``.\test\powershel``
|
||||
|
||||
``powershell
|
||||
|
||||
# To seed the test data
|
||||
.\test\powershell\test_retrievedata.ps1
|
||||
|
||||
|
||||
# For initial tests
|
||||
.\test\powershell\test_MartiLQ.ps1
|
||||
|
||||
#
|
||||
.\test\powershell\test_MartiLQCkan.ps1
|
||||
|
||||
#
|
||||
.\test\powershell\test_MartiLQData1.ps1
|
||||
|
||||
#
|
||||
.\test\powershell\test_MartiLQData2.ps1
|
||||
|
||||
#
|
||||
.\test\powershell\test_MartiLQData3.ps1
|
||||
|
||||
``
|
||||
|
|
@ -2,8 +2,8 @@
|
|||
# .\test\powershell\test_MartiLQCkan.ps1 from project root
|
||||
|
||||
. .\source\powershell\MartiLQ.ps1
|
||||
. .\source\powershell\MartiLQItem.ps1
|
||||
. .\source\powershell\ConvertFrom-Ckan.ps1
|
||||
. .\source\powershell\MartiLQResource.ps1
|
||||
. .\source\powershell\MartiLQUtilities.ps1
|
||||
|
||||
$outFile = ".\test\powershell\results\marti_test_asic.json"
|
||||
$ckan = Get-Content -Path ".\docs\source\samples\json\asic_ckan_api.json" -Raw
|
||||
|
|
|
|||
|
|
@ -1,8 +1,8 @@
|
|||
|
||||
. .\source\powershell\MartiLQ.ps1
|
||||
. .\source\powershell\MartiLQItem.ps1
|
||||
. .\source\powershell\ConvertFrom-Ckan.ps1
|
||||
. .\source\powershell\Compare-MartiLQResource.ps1
|
||||
. .\source\powershell\MartiLQResource.ps1
|
||||
. .\source\powershell\MartiLQAttribute.ps1
|
||||
. .\source\powershell\MartiLQUtilities.ps1
|
||||
|
||||
try {
|
||||
|
||||
|
|
|
|||
|
|
@ -1,8 +1,9 @@
|
|||
|
||||
. .\source\powershell\MartiLQ.ps1
|
||||
. .\source\powershell\MartiLQItem.ps1
|
||||
. .\source\powershell\ConvertFrom-Ckan.ps1
|
||||
. .\source\powershell\Compare-MartiLQResource.ps1
|
||||
. .\source\powershell\MartiLQResource.ps1
|
||||
. .\source\powershell\MartiLQAttribute.ps1
|
||||
. .\source\powershell\MartiLQUtilities.ps1
|
||||
|
||||
|
||||
try {
|
||||
|
||||
|
|
|
|||
|
|
@ -1,8 +1,8 @@
|
|||
|
||||
|
||||
|
||||
. .\source\powershell\New-Marti.ps1
|
||||
. .\source\powershell\ConvertFrom-Ckan.ps1
|
||||
. .\source\powershell\MartiLQ.ps1
|
||||
. .\source\powershell\MartiLQUtilities.ps1
|
||||
|
||||
|
||||
if (!(Test-Path -Path ".\test\powershell\results\data")) {
|
||||
|
|
|
|||
Loading…
Reference in New Issue