diff --git a/.gitignore b/.gitignore index 250f87d..e9a49a1 100644 --- a/.gitignore +++ b/.gitignore @@ -21,4 +21,6 @@ source/python/client/__pycache__ source/golang/client/src/test source/golang/client/src/martilq_client* -source/golang/client/src/*.exe \ No newline at end of file +source/golang/client/src/*.exe + +docs/site/ \ No newline at end of file diff --git a/docs/mkdocs.yml b/docs/mkdocs.yml new file mode 100644 index 0000000..299b32d --- /dev/null +++ b/docs/mkdocs.yml @@ -0,0 +1,28 @@ +site_name: martiLQ +site_url: https://martilq.readthedocs.com/ + +docs_dir: source +nav: + - Home: README.md + - martiLQ: martiLQ.md + - Why: why.md + - What: what.md + - When: when.md + - Who: who.md + - Resource: resources.md + - Attributes: attributes.md + - Custom: custom.md + - Quality: quality.md + - Samples: samples\json\README.md + - Comparison: comparison.md + - CKAN: ckan.md + - MAGDA: magda.md + - References: references.md + +theme: readthedocs +extra_css: + - stylesheets/extra.css + +site_author: meerkat + +repo_url: https://github.com/meerkat-manor/marti/ diff --git a/docs/source/README.md b/docs/source/README.md index 36dc073..0fa71a4 100644 --- a/docs/source/README.md +++ b/docs/source/README.md @@ -9,20 +9,35 @@ documents. There is no quickstart document to get you started as each use case and organisation is different. There are sample implementations which you can adjust if they resonate with your circumstances, - see [sample implementations](samples/) +see [sample implementations](samples/) -## Index -1. [Objective](objective.md) -2. [martiLQ](martiLQ.md) -3. [Why you would use martiLQ](why.md) -4. [What is martiLQ](what.md) -5. [When would you start using martiLQ](when.md) -6. [Who would use martiLQ](who.md) -7. [Resources definition](resources.md) -8. [Attributes definition](attributes.md) -9. [Custom definition](custom.md) -10. [Load quality metrics support](quality.md) -11. [Comparison of martiLQ definition](comparison.md) -12. [References](references.md) +There are sample implementations which you +can adjust if they resonate with your circumstances. +The source, documentation and samples are available at ``_ + +## MartiLQ objective + +The objective of **martiLQ** is to define a simple standard for +capturing the data files being transferred. It is not for +real time web service transactions. + +**martiLQ** is about file and document transfer and reconciling +that the all files have arrived and have not changed, and if so +required are also encrypted. + +The proposition is to have a common, machine readable format +for file exchange that: + + * ensures data load quality and reconciles + * can be used on Linux or Windows or Mac + * can be used with Python, Java, PowerShell, Golang, etc + * can be used by web services + * uses a text based format (JSON) + * can form part of the data processing pipeline + +And finally is easy to understand. + +To get a better understanding have a look at the definition +in [martiLQ](martiLQ.md) diff --git a/docs/source/attributes.md b/docs/source/attributes.md index 6b67554..8888c99 100644 --- a/docs/source/attributes.md +++ b/docs/source/attributes.md @@ -1,22 +1,24 @@ -Attribute definition -==================== +# Attribute definition A Resource can list attributes related to the document / file. An attribute is a generic definition and conventions are -observed in the definitions that are captured here. +observed in the definitions that are captured here. The attribute +section is where load quality metrics are defined. -Attribute definition --------------------- +## Attribute definition -The Attribute consists of: +The Attribute is described by the table below. Recommended +values are listed but custom values can also be defined, just be +certain the recipient is able to understand them. - * category - A value of "dataset", - * name - * function - A value such as "count" - * comparison - A comparisn value or NA. Values are "EQ", - "NE", "GT", "LT - * value - The value for the attribute based on the above complex key, excluding comparison +Name|Description|Values or Default +---|---|--- +category|A type of attribute|dataset, format +name|A name for the attribute|records,columns,header,footer,separator,quote, escape +function|A function to perform|count,sum,unique +comparison|A comparison value or NA|NA, EQ, NE, GT, GE, LT,LE +value|The value for the attribute based on the above complex key, excluding comparison|numeric A sample JSON is shown below which describes the number of records in the file for the given format. diff --git a/docs/source/ckan.md b/docs/source/ckan.md index cf97138..69f6c3c 100644 --- a/docs/source/ckan.md +++ b/docs/source/ckan.md @@ -1,16 +1,15 @@ -CKAN definition -=============== +# CKAN definition The **martiLQ** has used similar terms and structures that are found in the CKAN API describing the resources. This similarity allows for simple mapping of values from the CKAN format to **martiLQ** format. -What **martiLQ** brings above the CKAN definition is attributes that can be -used to reconcile with the recieved data, plus the ability to define +What **martiLQ** extends above the CKAN definition is attributes that can be +used to reconcile with the received data, plus the ability to define compressed and encrypted resources. For more information on CKAN see https://ckan.org/ -A sample Json to compare against the **martiLQ** definition +A sample JSON to compare against the **martiLQ** document is https://data.gov.au/data/dataset/f2b7c2c1-f4ef-4ae9-aba5-45c19e4d3038 diff --git a/docs/source/conf.py b/docs/source/conf.py deleted file mode 100644 index 418a854..0000000 --- a/docs/source/conf.py +++ /dev/null @@ -1,65 +0,0 @@ -# Configuration file for the Sphinx documentation builder. -# -# This file only contains a selection of the most common options. For a full -# list see the documentation: -# https://www.sphinx-doc.org/en/master/usage/configuration.html - -# -- Path setup -------------------------------------------------------------- - -# If extensions (or modules to document with autodoc) are in another directory, -# add these directories to sys.path here. If the directory is relative to the -# documentation root, use os.path.abspath to make it absolute, like shown here. -# -# import os -# import sys -# sys.path.insert(0, os.path.abspath('.')) -from recommonmark.parser import CommonMarkParser - -# -- Project information ----------------------------------------------------- - -project = 'MartiLQ' -copyright = '2021, meerkat@merebox.coom' -author = 'meerkat@merebox.coom' - -# The full version, including alpha/beta/rc tags -release = '0.0.1' - - -# -- General configuration --------------------------------------------------- - -# Add any Sphinx extension module names here, as strings. They can be -# extensions coming with Sphinx (named 'sphinx.ext.*') or your custom -# ones. -#extensions = ['myst_parser'] - -# Add any paths that contain templates here, relative to this directory. -templates_path = ['_templates'] - -# List of patterns, relative to source directory, that match files and -# directories to ignore when looking for source files. -# This pattern also affects html_static_path and html_extra_path. -exclude_patterns = [ - 'samples/python/test/*', - 'samples/powershell/test/*', - ] - -source_parsers = { - '.md': CommonMarkParser -} - -source_suffix = ['.rst', '.md'] - - -# -- Options for HTML output ------------------------------------------------- - -# The theme to use for HTML and HTML Help pages. See the documentation for -# a list of builtin themes. -# -#html_theme = 'alabaster' -html_theme = 'sphinx_rtd_theme' - - -# Add any paths that contain custom static files (such as style sheets) here, -# relative to this directory. They are copied after the builtin static files, -# so a file named "default.css" will overwrite the builtin "default.css". -html_static_path = ['_static'] \ No newline at end of file diff --git a/docs/source/index.rst b/docs/source/index.rst deleted file mode 100644 index 8d1e280..0000000 --- a/docs/source/index.rst +++ /dev/null @@ -1,37 +0,0 @@ -.. MartiLQ documentation master file, created by - sphinx-quickstart on Sun Oct 31 14:30:06 2021. - You can adapt this file completely to your liking, but it should at least - contain the root `toctree` directive. - -Welcome to MartiLQ's documentation! -=================================== - -.. toctree:: - :hidden: - :maxdepth: 2 - - objective.md - martiLQ.md - why.md - what.md - when.md - who.md - resources.md - attributes.md - custom.md - quality.md - comparison.md - ckan.md - magda.md - references.md - -**martiLQ** stands for metadata reconcilation for transfer information, load quality. - -Before starting with **martiLQ** it is advisable to understand if it is right for -your organisation's needs. Information is available in a number of short -documents. Start with the (objectives)[objective.md] - -There are sample implementations which you -can adjust if they resonate with your circumstances. - -The source, documentation and samples are available at ``_ diff --git a/docs/source/martiLQ.md b/docs/source/martiLQ.md index 2bc1166..46d5809 100644 --- a/docs/source/martiLQ.md +++ b/docs/source/martiLQ.md @@ -1,33 +1,47 @@ -MartiLQ document -================ +# MartiLQ document The metadata reconciliation transfer information is referred to as the **martiLQ** document throughout this documentation. The **martiLQ** document can be part of a message or a file -in its own right. The definition is curently a Json file. +in its own right. The definition is currently a JSON file. -Mandatory information ---------------------- +## Structure -The mandatory information is: +The JSON is composed of: -* Title -* Unique identifier -* Resource list - See Resource section summary below or detailed document [Resource](resources.md) +* A root definition that contains information applicable to all reosurces +* A [resource](resources.md) list that contains information related + to each document or file +* [Attribute](attributes.md) list as a child to each resource +* A [custom](custom.md) list -Optional information -++++++++++++++++++++ +## Mandatory root information -The optional information is: +The mandatory information in the root of the **martiLQ** document is: + +Name|Description|Default or values +---|---|--- +title|A title or batch identifier. Use a name that is easy to understand or relate to|None +uid|Unique identifier for the document. If the same document is reproduced this may not change but the minor number of the batch must change|Auto generated +resources|Resource list. See Resource section summary below or detailed document [Resource](resources.md)|At least one required + +### Optional information + + +The optional information is described in the table below. A number of elements can be configured for their +default value(s) in a configuration file. + +Name|Description|Default or values +---|---|-- +description|Long description of the purpose, background or files included|None +modified|Modified date and time of the **martiLQ** document|Now +tags|List of tags or keywords| +publisher|Publisher name| +contactPoint|Contact point of a person or team| +accessLevel|Acces level| +rights|Rights| -* Description -* Modified -* Tags or keywords -* Publisher -* Contact point -* Acces level -* Rights * Batch * License * Described By - A link to the metadata describing the document. @@ -37,14 +51,12 @@ The optional information is: * Custom list - List of custom entries, one being the **martiLQ** software details see [custom](custom.md) -Information extension -+++++++++++++++++++++ +### Information extension The information supplied can be extended by party agreement and there are place holders in the defintion. -Resource --------- +## Resource The resource section is a list of documents or files that are to be grouped together are listed under the same **martiLQ** definition. @@ -54,26 +66,107 @@ it will commonly be for definiting multiple formats, with each file having a different extension. Commonly the definition includes at least the following items: -* Title - A summary description of the document contents -* Unique identifier - A unique identifier, commonly a GUID -* Document name - A name of the document such as thefile name -* Issued date - When the document was made available. The date can include time -* Modified - When the document was created or modified. This is the data and time -* Size of document - The document size in bytes -* URL - This can be ``file://``, ``https://``, ``ftp://``, etc resource location +Name|Description|Default or values +---|---|-- +title|Title for the resource|Document name +uid|A unique identifier, commonly a GUID|Auto generated +documentName|A name of the document such as the file name +issueDate|Issued date - When the document was made available. The date can include time +modified|Modified - When the document was created or modified. This is the data and time +size|Size of document - The document size in bytes +url|URL - This can be ``file://``, ``https://``, ``ftp://``, etc resource location -Resource optional -+++++++++++++++++ +### Resource optional The following are some of the optional items in the resource section. See [Resource](resources.md) for more details -* Hash of document - The hash of the document, which can be blank especially for large documents -* Hash algorithm - Algoroithm used to generate the hash value or sign it -* Description - A more detailed description -* Version - A document version -* Encoding -* Content Type -* Compression -* Encryption -* Author +Name|Description|Default or values +---|---|-- +hash|Hash of document - The hash of the document, which can be blank especially for large documents +algo|Hash algorithm - Algoroithm used to generate the hash value or sign it +description|Description - A more detailed description +version|Version - A document version +encoding|Encoding +contentType|Content Type +compression|Compression|None +encryption|Encryption|None +author|Author +attributes|List of attributes for the resource|Record count + +## Simple sample + +A sample of a single resource **martiLQ** document is shown below. The +sample can be generated using the GOLANG client program with parameters: + +``` +-t GEN -m Sample.json -s ./docs/source/martilq.md --title "GEN001" --description "Simple example" +``` + +```json +{ + "content-type": "application/vnd.martilq.json", + "title": "GEN001", + "uid": "9a0a7edb-dd81-4fc5-a6cb-c5716eda7b51", + "description": "Simple example", + "modified": "2021-11-02T22:44:29.6887001+11:00", + "publisher": "", + "contactPoint": "", + "accessLevel": "Confidential", + "rights": "Restricted", + "tags": null, + "license": "", + "state": "active", + "batch": 1.001, + "describedBy": "", + "landingPage": "", + "theme": "", + "resources": [ + { + "title": "martilq.md", + "uid": "a88b4e5f-66b7-4003-ac24-831c95d0da07", + "documentName": "martilq.md", + "issueDate": "2021-11-02T22:44:29.6881663+11:00", + "modified": "2021-11-02T07:47:13.9410018+11:00", + "expires": "2023-11-02T00:00:00+11:00", + "state": "active", + "author": "", + "length": 3654, + "hash": { + "algo": "SHA256", + "value": "213a6254ddc02423b6c3bb3d977892678258539d37f06410ef18d27c14ffa821", + "signed": false + }, + "description": "", + "url": "http://localhost/martilq/martilq.md", + "version": "", + "content-type": "", + "encoding": "UTF-8", + "compression": "", + "encryption": "", + "describedBy": "", + "attributes": [ + { + "category": "dataset", + "name": "records", + "function": "count", + "comparison": "EQ", + "value": "95" + } + ] + } + ], + "custom": [ + { + "extension": "software", + "softwareName": "MARTILQREFERENCE", + "author": "Meerkat@merebox.com", + "version": "0.0.1" + } + ] +} +``` + +You can view a more complete sample [samples/json/sample_02.md](samples/json/sample_02.md) +which has been generated using a configuration file to supply default values. + diff --git a/docs/source/objective.md b/docs/source/objective.md deleted file mode 100644 index 8c9b832..0000000 --- a/docs/source/objective.md +++ /dev/null @@ -1,23 +0,0 @@ -MartiLQ objective -================= - -The objective of **martLQ** is to define a simle standard for -capturing the data files being transferred. It is not for -real time web service transactions. - -**martiLQ** is about file and document transfer and reconciling -that the all files have arrived and have not changed, and if so -required are also encrypted. - -The proposition is to have a common, machine readable format -for file exchange that: - - * ensures data load quality and reconciles - * can be used on Linux or Windows or Mac - * can be used with Python, Java, PowerShell, Golang, etc - * can be used by web services - * uses a text based format (JSON) - * can form part of the data processing pipeline - -And finally is easy to understand. - diff --git a/docs/source/quality.md b/docs/source/quality.md index f843d01..8b84489 100644 --- a/docs/source/quality.md +++ b/docs/source/quality.md @@ -1,16 +1,18 @@ -Quality definition -================== +# Load Quality -The **martiLQ** definition allows for the inclusion of load quality -definitions. The load quality definition is intended to be +The **martiLQ** document allows for the inclusion of load quality +metrics. The load quality metrics is intended to be able to be applied universally with common tools. Not all needs are covered with the base definition but can be extended. -Defined load quality metrics ----------------------------- +The load quality metrics are in the majority defined in the [attributes](attributes.md) +list attached to each resource. Therefore each resource can have different +load quality metrics. -* Sequential batch number - This is a decimal number defined at the **martiLQ** definition - and applies to all resources. The integer portion is for new batches and the fraction +## Defined load quality metrics + +* Sequential batch number - This is a decimal number defined at the **martiLQ** document + header and applies to all resources. The integer portion is for new batches and the fraction part can be used for issues with the same data extract. such as requiring resend because a resource was missing. @@ -19,8 +21,7 @@ Defined load quality metrics on the number of primary segments under root. JSON records can be counted in a similar way. The headers or trailling records are not counted -Addresses deficiencies ----------------------- +## Addresses deficiencies The **martiLQ** objective is to address deficiencies with alternative data load quality approaches such as: @@ -32,10 +33,9 @@ data load quality approaches such as: * securing the data * adding footers to the data, requiring custom file handlers -Extending load quality metrics ------------------------------- +## Extending load quality metrics -**martLQ** framewokr and importantly is open to extension so that extra +**martLQ** document is open to extension so that extra load metrics appropriate to the situation can be included. ### Extension ideas @@ -47,5 +47,5 @@ The following extensions for load quality can easily be included: * Data values are within defined tolerances * Check for data exclusions -And all this information is included in the **martiLQ** definition +And all this information is included in the **martiLQ** document allowing for self describing load quality. diff --git a/docs/source/references.md b/docs/source/references.md index 207e89d..9e0d285 100644 --- a/docs/source/references.md +++ b/docs/source/references.md @@ -1,6 +1,7 @@ -References -========== +# References +The following are references to documents that inspired the creation of **martiLQ** +document and associatd framework. https://dex.dss.gov.au/sites/default/files/documents/2021-06/data-exchange-protocols-june-2021.pdf diff --git a/docs/source/resources.md b/docs/source/resources.md index 09d6497..2219724 100644 --- a/docs/source/resources.md +++ b/docs/source/resources.md @@ -1,5 +1,4 @@ -Resources definition -==================== +# Resources definition The resources section defines the files that are grouped together by association. This association is not defined but can @@ -39,8 +38,7 @@ The following are optional in the resource section. * Encryption - Type of encryption used if any -Compression ------------ +## Compression Files can be compressed using a utility. A single compressed file can contain multiple files. The **martiLQ** definition document applies to the compressed file @@ -51,8 +49,7 @@ compressed file. Compression of files always occur before encryption. -martiLQ definition for Compressed File -++++++++++++++++++++++++++++++++++++++ +### martiLQ definition for Compressed File For a compressed file that is not encrypted, the distribution definition will be: @@ -71,8 +68,7 @@ The reason for this approach is it allows a generic tool to be deployed to check the validity of the contents without unpacking the received /fetched file. That is you can perform load quality pipeline processing. -Encryption ----------- +## Encryption The encryption of content is always applied after compression not before, if you are not using the compression tool native encryption. WinZIP and 7ZIP diff --git a/docs/source/samples/json/README.md b/docs/source/samples/json/README.md new file mode 100644 index 0000000..d00ccec --- /dev/null +++ b/docs/source/samples/json/README.md @@ -0,0 +1,11 @@ +# Samples + +This folder contains a number of sample **martiLQ** documents +or configuration files. + +File|Content +---|--- +[sample_01.md](sample_01.md)|Basic output +[sample_02.md](sample_02.md)|Basic ouput with configuration file +[sample_03.md](sample_03.md)|Directory output + diff --git a/docs/source/samples/json/martilq_02.ini b/docs/source/samples/json/martilq_02.ini new file mode 100644 index 0000000..50580bd --- /dev/null +++ b/docs/source/samples/json/martilq_02.ini @@ -0,0 +1,55 @@ + +[General] + +logPath = +tempPath = +dataPath = + +[MartiLQ] + +tags = sample +publisher = meerkat@merebox.com +contactPoint = Your friendly Meerkat +accessLevel = Confidential +rights = Public +license = MIT +batch = @./config/batch.no +theme = Documentation + + +[Resources] + +author = Hive +title = {{documentName}} +state = expired +expires = 2:0:0 +encoding = UTF-8 +version = 1.0 +urlPrefix = http://localhost/martilq/ + +[Hash] + +hashAlgorithm = +signKey_File = +signKey_Password = + + +[Network] + +proxy = +username = +password = + + +[Custom_Spatial] + +enabled = true +country = Netherland +region = +town = Amsterdam + +[Custom_Temporal] + +enabled = true +businessDate = {{yesterday}} +runDate = {{today}} diff --git a/docs/source/samples/json/sample_01.md b/docs/source/samples/json/sample_01.md new file mode 100644 index 0000000..37362af --- /dev/null +++ b/docs/source/samples/json/sample_01.md @@ -0,0 +1,3 @@ +# Basic sample + + diff --git a/docs/source/samples/json/sample_02.md b/docs/source/samples/json/sample_02.md new file mode 100644 index 0000000..c050a27 --- /dev/null +++ b/docs/source/samples/json/sample_02.md @@ -0,0 +1,122 @@ +# Sample with configuration file + +The sample **martiLQ** document below has been generated +using the client GOLANG program and a configuration file +with changed values from the default + +The source is the project folder /docs/source + +The configuration file is + +```ini + +[MartiLQ] +tags = sample +publisher = meerkat@merebox.com +contactPoint = Your friendly Meerkat +accessLevel = Confidential +rights = Public +license = MIT +batch = @./config/batch.no +theme = Documentation + +[Resources] +author = Hive +title = {{documentName}} +state = expired +expires = 2:0:0 +encoding = UTF-8 +urlPrefix = http://localhost/martilq/ + +[Custom_Spatial] +enabled = true +country = Netherland +region = +town = Amsterdam + +[Custom_Temporal] +enabled = true +businessDate = {{yesterday}} +runDate = {{today}} + +``` + +The **martiLQ** document + +```json +{ + "content-type": "application/vnd.martilq.json", + "title": "GEN001", + "uid": "369fb6c4-3628-4f61-a3ee-7b03a0fc8e25", + "description": "Simple example", + "modified": "2021-11-02T22:56:03.5897714+11:00", + "publisher": "meerkat@merebox.com", + "contactPoint": "Your friendly Meerkat", + "accessLevel": "Confidential", + "rights": "Public", + "tags": [ + "sample" + ], + "license": "MIT", + "state": "expired", + "batch": 1.001, + "describedBy": "", + "landingPage": "", + "theme": "Documentation", + "resources": [ + { + "title": "martilq.md", + "uid": "b0206363-5dcb-485d-83e9-9495e75662c1", + "documentName": "martilq.md", + "issueDate": "2021-11-02T22:56:03.5892511+11:00", + "modified": "2021-11-02T22:47:49.7108132+11:00", + "expires": "2023-11-02T00:00:00+11:00", + "state": "expired", + "author": "Hive", + "length": 5873, + "hash": { + "algo": "SHA256", + "value": "38714907ced5ff5efbf939f6404634dfa51762cec075b82187c0c6fa880aa37b", + "signed": false + }, + "description": "", + "url": "http://localhost/martilq/martilq.md", + "version": "", + "content-type": "", + "encoding": "UTF-8", + "compression": "", + "encryption": "", + "describedBy": "", + "attributes": [ + { + "category": "dataset", + "name": "records", + "function": "count", + "comparison": "EQ", + "value": "169" + } + ] + } + ], + "custom": [ + { + "extension": "software", + "softwareName": "MARTILQREFERENCE", + "author": "Meerkat@merebox.com", + "version": "0.0.1" + }, + { + "extension": "spatial", + "country": "Netherland", + "region": "", + "town": "Amsterdam" + }, + { + "extension": "temporal", + "businessDate": "2021-11-01T00:00:00+11:00", + "runDate": "2021-11-02T00:00:00+11:00" + } + ] +} +``` + diff --git a/docs/source/samples/json/sample_03.md b/docs/source/samples/json/sample_03.md new file mode 100644 index 0000000..bff6d83 --- /dev/null +++ b/docs/source/samples/json/sample_03.md @@ -0,0 +1,515 @@ +# Sample with entire directory + +The sample **martiLQ** document below has been generated +using the client GOLANG program and including all files +in a directory. + +The source is the project folder /docs/source and is recursive +to include child folders and documents. + + +```json +{ + "content-type": "application/vnd.martilq.json", + "title": "GEN003", + "uid": "67152b70-368e-4186-ab06-398f4c98a079", + "description": "Directory example", + "modified": "2021-11-02T23:48:07.0787735+11:00", + "publisher": "meerkat@merebox.com", + "contactPoint": "Your friendly Meerkat", + "accessLevel": "Confidential", + "rights": "Public", + "tags": [ + "sample" + ], + "license": "MIT", + "state": "expired", + "batch": 1.001, + "describedBy": "", + "landingPage": "", + "theme": "Documentation", + "resources": [ + { + "title": "README.md", + "uid": "e32b8a62-5571-41a5-beba-071844a80082", + "documentName": "README.md", + "issueDate": "2021-11-02T23:48:07.0480017+11:00", + "modified": "2021-11-01T22:33:07.3888826+11:00", + "expires": "2023-11-02T00:00:00+11:00", + "state": "expired", + "author": "Hive", + "length": 1566, + "hash": { + "algo": "SHA256", + "value": "cf7cbf4186779984144fd022f4ba77b9773ffdc799b4a87f74e0ad2c52a7a261", + "signed": false + }, + "description": "", + "url": "http://localhost/martilq/README.md", + "version": "", + "content-type": "", + "encoding": "UTF-8", + "compression": "", + "encryption": "", + "describedBy": "", + "attributes": [ + { + "category": "dataset", + "name": "records", + "function": "count", + "comparison": "EQ", + "value": "43" + } + ] + }, + { + "title": "attributes.md", + "uid": "2d431e9b-9906-44bc-8b0b-6746ca9e5c4c", + "documentName": "attributes.md", + "issueDate": "2021-11-02T23:48:07.0497323+11:00", + "modified": "2021-11-01T22:56:49.7231238+11:00", + "expires": "2023-11-02T00:00:00+11:00", + "state": "expired", + "author": "Hive", + "length": 1204, + "hash": { + "algo": "SHA256", + "value": "38b27aa1302da2abfb1ed6e654f93681571158bb28586919e4cdf771d49d2ed5", + "signed": false + }, + "description": "", + "url": "http://localhost/martilq/attributes.md", + "version": "", + "content-type": "", + "encoding": "UTF-8", + "compression": "", + "encryption": "", + "describedBy": "", + "attributes": [ + { + "category": "dataset", + "name": "records", + "function": "count", + "comparison": "EQ", + "value": "36" + } + ] + }, + { + "title": "ckan.md", + "uid": "4141a67a-dcb3-4b0c-b819-66555d0699b4", + "documentName": "ckan.md", + "issueDate": "2021-11-02T23:48:07.0509128+11:00", + "modified": "2021-11-01T22:59:14.8560535+11:00", + "expires": "2023-11-02T00:00:00+11:00", + "state": "expired", + "author": "Hive", + "length": 613, + "hash": { + "algo": "SHA256", + "value": "17e886292b873937f4ff6bb18c32196ffd19311c4adb00ad989d13d01bd738d7", + "signed": false + }, + "description": "", + "url": "http://localhost/martilq/ckan.md", + "version": "", + "content-type": "", + "encoding": "UTF-8", + "compression": "", + "encryption": "", + "describedBy": "", + "attributes": [ + { + "category": "dataset", + "name": "records", + "function": "count", + "comparison": "EQ", + "value": "15" + } + ] + }, + { + "title": "comparison.md", + "uid": "b76ca118-b01e-4f26-9395-c5c07051d0e2", + "documentName": "comparison.md", + "issueDate": "2021-11-02T23:48:07.0521334+11:00", + "modified": "2021-11-01T14:27:20.7769396+11:00", + "expires": "2023-11-02T00:00:00+11:00", + "state": "expired", + "author": "Hive", + "length": 1974, + "hash": { + "algo": "SHA256", + "value": "2c11419e22124990c68399fd332bb42508221cd6daeb4bc61b25ab44a12e6d68", + "signed": false + }, + "description": "", + "url": "http://localhost/martilq/comparison.md", + "version": "", + "content-type": "", + "encoding": "UTF-8", + "compression": "", + "encryption": "", + "describedBy": "", + "attributes": [ + { + "category": "dataset", + "name": "records", + "function": "count", + "comparison": "EQ", + "value": "46" + } + ] + }, + { + "title": "custom.md", + "uid": "e6829ac7-e0bd-413b-b035-2cf2aee1da5d", + "documentName": "custom.md", + "issueDate": "2021-11-02T23:48:07.0532497+11:00", + "modified": "2021-11-01T13:52:22.4420815+11:00", + "expires": "2023-11-02T00:00:00+11:00", + "state": "expired", + "author": "Hive", + "length": 397, + "hash": { + "algo": "SHA256", + "value": "c55ca715521b98fdad99e020c9a983d694557db84a25a9b5e2b0fbce7ece3549", + "signed": false + }, + "description": "", + "url": "http://localhost/martilq/custom.md", + "version": "", + "content-type": "", + "encoding": "UTF-8", + "compression": "", + "encryption": "", + "describedBy": "", + "attributes": [ + { + "category": "dataset", + "name": "records", + "function": "count", + "comparison": "EQ", + "value": "10" + } + ] + }, + { + "title": "magda.md", + "uid": "39879fe6-3525-4152-9b93-acf20aefd683", + "documentName": "magda.md", + "issueDate": "2021-11-02T23:48:07.0554587+11:00", + "modified": "2021-11-01T14:26:31.8900363+11:00", + "expires": "2023-11-02T00:00:00+11:00", + "state": "expired", + "author": "Hive", + "length": 120, + "hash": { + "algo": "SHA256", + "value": "74dcb357989485a7e98ab5c2b718460a0bb81c50b4c3afd8aa72091c4cd62e48", + "signed": false + }, + "description": "", + "url": "http://localhost/martilq/magda.md", + "version": "", + "content-type": "", + "encoding": "UTF-8", + "compression": "", + "encryption": "", + "describedBy": "", + "attributes": [ + { + "category": "dataset", + "name": "records", + "function": "count", + "comparison": "EQ", + "value": "8" + } + ] + }, + { + "title": "martiLQ.md", + "uid": "d259ef9f-38a4-4af2-bfdb-baa29e5ea8c9", + "documentName": "martiLQ.md", + "issueDate": "2021-11-02T23:48:07.0569604+11:00", + "modified": "2021-11-02T22:57:00.0061831+11:00", + "expires": "2023-11-02T00:00:00+11:00", + "state": "expired", + "author": "Hive", + "length": 6046, + "hash": { + "algo": "SHA256", + "value": "111f3ad34d94dc346ac282c9a1cc9a3e5802706b5274684b660d2f9b1721abcf", + "signed": false + }, + "description": "", + "url": "http://localhost/martilq/martiLQ.md", + "version": "", + "content-type": "", + "encoding": "UTF-8", + "compression": "", + "encryption": "", + "describedBy": "", + "attributes": [ + { + "category": "dataset", + "name": "records", + "function": "count", + "comparison": "EQ", + "value": "172" + } + ] + }, + { + "title": "quality.md", + "uid": "08966d7f-3f6b-4187-9fb8-432fa78cb43d", + "documentName": "quality.md", + "issueDate": "2021-11-02T23:48:07.0581721+11:00", + "modified": "2021-11-01T22:43:35.410177+11:00", + "expires": "2023-11-02T00:00:00+11:00", + "state": "expired", + "author": "Hive", + "length": 2019, + "hash": { + "algo": "SHA256", + "value": "e87c546fae45363d9cd1db0228885290aecc89105c238d203d41253f08d1699f", + "signed": false + }, + "description": "", + "url": "http://localhost/martilq/quality.md", + "version": "", + "content-type": "", + "encoding": "UTF-8", + "compression": "", + "encryption": "", + "describedBy": "", + "attributes": [ + { + "category": "dataset", + "name": "records", + "function": "count", + "comparison": "EQ", + "value": "51" + } + ] + }, + { + "title": "references.md", + "uid": "842066e0-2e81-487b-9f48-0df972aa6f12", + "documentName": "references.md", + "issueDate": "2021-11-02T23:48:07.0593388+11:00", + "modified": "2021-11-01T23:25:15.048493+11:00", + "expires": "2023-11-02T00:00:00+11:00", + "state": "expired", + "author": "Hive", + "length": 722, + "hash": { + "algo": "SHA256", + "value": "659db48c3b80a7c1ca3443fc43c45fe76eaa09c151e15f4dc1825fc84653f065", + "signed": false + }, + "description": "", + "url": "http://localhost/martilq/references.md", + "version": "", + "content-type": "", + "encoding": "UTF-8", + "compression": "", + "encryption": "", + "describedBy": "", + "attributes": [ + { + "category": "dataset", + "name": "records", + "function": "count", + "comparison": "EQ", + "value": "24" + } + ] + }, + { + "title": "resources.md", + "uid": "21659556-bf8b-486e-868f-c739f3f9cee8", + "documentName": "resources.md", + "issueDate": "2021-11-02T23:48:07.0604619+11:00", + "modified": "2021-11-01T23:23:55.517501+11:00", + "expires": "2023-11-02T00:00:00+11:00", + "state": "expired", + "author": "Hive", + "length": 4646, + "hash": { + "algo": "SHA256", + "value": "d2918d7b373a8b3bf43f377bc3d8df21f623b3c2a61bf5a16e19602fa763b8fc", + "signed": false + }, + "description": "", + "url": "http://localhost/martilq/resources.md", + "version": "", + "content-type": "", + "encoding": "UTF-8", + "compression": "", + "encryption": "", + "describedBy": "", + "attributes": [ + { + "category": "dataset", + "name": "records", + "function": "count", + "comparison": "EQ", + "value": "98" + } + ] + }, + { + "title": "what.md", + "uid": "60b0ddad-d860-415e-a3da-4fdfb9d7dc91", + "documentName": "what.md", + "issueDate": "2021-11-02T23:48:07.0638034+11:00", + "modified": "2021-11-01T23:22:55.6854301+11:00", + "expires": "2023-11-02T00:00:00+11:00", + "state": "expired", + "author": "Hive", + "length": 864, + "hash": { + "algo": "SHA256", + "value": "d80c5445b62a2aefd670b0a5fd6f25031f67cc858a1a9826afdbfb82985c6450", + "signed": false + }, + "description": "", + "url": "http://localhost/martilq/what.md", + "version": "", + "content-type": "", + "encoding": "UTF-8", + "compression": "", + "encryption": "", + "describedBy": "", + "attributes": [ + { + "category": "dataset", + "name": "records", + "function": "count", + "comparison": "EQ", + "value": "18" + } + ] + }, + { + "title": "when.md", + "uid": "add6d143-e602-4066-b804-3ce7fc1214e4", + "documentName": "when.md", + "issueDate": "2021-11-02T23:48:07.0650335+11:00", + "modified": "2021-11-01T23:22:49.8615445+11:00", + "expires": "2023-11-02T00:00:00+11:00", + "state": "expired", + "author": "Hive", + "length": 1106, + "hash": { + "algo": "SHA256", + "value": "605c97bb103d352c6d8fefe5c86c7e521b5998048a9ae876d6f4c8cd551ec9d1", + "signed": false + }, + "description": "", + "url": "http://localhost/martilq/when.md", + "version": "", + "content-type": "", + "encoding": "UTF-8", + "compression": "", + "encryption": "", + "describedBy": "", + "attributes": [ + { + "category": "dataset", + "name": "records", + "function": "count", + "comparison": "EQ", + "value": "22" + } + ] + }, + { + "title": "who.md", + "uid": "f281360b-b9ed-4485-b0d7-20f92a677085", + "documentName": "who.md", + "issueDate": "2021-11-02T23:48:07.0748937+11:00", + "modified": "2021-11-01T14:28:37.7765182+11:00", + "expires": "2023-11-02T00:00:00+11:00", + "state": "expired", + "author": "Hive", + "length": 1943, + "hash": { + "algo": "SHA256", + "value": "c28ece0930b5671896b8ada02b13e85aa5c050d824c3843d557e6ef48579df55", + "signed": false + }, + "description": "", + "url": "http://localhost/martilq/who.md", + "version": "", + "content-type": "", + "encoding": "UTF-8", + "compression": "", + "encryption": "", + "describedBy": "", + "attributes": [ + { + "category": "dataset", + "name": "records", + "function": "count", + "comparison": "EQ", + "value": "41" + } + ] + }, + { + "title": "why.md", + "uid": "f4e8b691-3117-4864-a4a9-24737b7fcad0", + "documentName": "why.md", + "issueDate": "2021-11-02T23:48:07.0771793+11:00", + "modified": "2021-11-01T14:29:02.7419541+11:00", + "expires": "2023-11-02T00:00:00+11:00", + "state": "expired", + "author": "Hive", + "length": 1614, + "hash": { + "algo": "SHA256", + "value": "1dbf3221b4a3bdb39e073e9e75dd3218bb45ae6e083bf5b02f63481e58fc8f91", + "signed": false + }, + "description": "", + "url": "http://localhost/martilq/why.md", + "version": "", + "content-type": "", + "encoding": "UTF-8", + "compression": "", + "encryption": "", + "describedBy": "", + "attributes": [ + { + "category": "dataset", + "name": "records", + "function": "count", + "comparison": "EQ", + "value": "37" + } + ] + } + ], + "custom": [ + { + "extension": "software", + "softwareName": "MARTILQREFERENCE", + "author": "Meerkat@merebox.com", + "version": "0.0.1" + }, + { + "extension": "spatial", + "country": "Netherland", + "region": "", + "town": "Amsterdam" + }, + { + "extension": "temporal", + "businessDate": "2021-11-01T00:00:00+11:00", + "runDate": "2021-11-02T00:00:00+11:00" + } + ] +} +``` diff --git a/docs/source/stylesheets/extra.css b/docs/source/stylesheets/extra.css new file mode 100644 index 0000000..e69de29 diff --git a/docs/source/what.md b/docs/source/what.md index db9990f..199d925 100644 --- a/docs/source/what.md +++ b/docs/source/what.md @@ -1,12 +1,11 @@ -What is marti -============= +# What is marti The foundation pillar for the **martiLQ** framework is the [martiLQ document](martiLQ.md) that defines the reconciliation and other metadata of the document / file being transferred. A definition, while fundamental, benefits from having tools that can create, read and interpret the definition. The **martiLQ** framework is about providing those tools -and a ecosystem that can be added to. +and an ecosystem that can be added to. The majority of the effort in creating **martiLQ** is in the tools. There are tools for various programming languages and situations. As many programming languages diff --git a/docs/source/when.md b/docs/source/when.md index 27ef352..9e7f2ae 100644 --- a/docs/source/when.md +++ b/docs/source/when.md @@ -1,5 +1,4 @@ -When would you use martiLQ -========================== +# When would you use martiLQ You are likely to start using the **martiLQ** framework when: @@ -12,8 +11,7 @@ If you already have a standard and it works for you, and you have no upcoming (l initiative that would benefit from the framework, then stick with what you have. The benefits of the framework are unlikely to weigh in your framework. -Read the material ------------------ +## Read the material Please read the material before jumping in and make sure the **martiLQ** framework will provide you with the expected benefits. Implementing the framework and then changing diff --git a/source/golang/client/src/config/martilq.ini b/source/golang/client/src/config/martilq.ini index 36ba8a7..b9e6fd8 100644 --- a/source/golang/client/src/config/martilq.ini +++ b/source/golang/client/src/config/martilq.ini @@ -7,21 +7,21 @@ dataPath = [MartiLQ] -tags = test,sample -publisher = meerkat@merebox.com -contactPoint = Meerkat -accessLevel = Confidential -rights = None -license = MIT +tags = +publisher = +contactPoint = +accessLevel = +rights = +license = batch = @./config/batch.no -theme = GOLANG +theme = [Resources] -author = Mobidick +author = title = {{documentName}} -state = active +state = expires = 2:0:0 encoding = UTF-8 version = @@ -43,13 +43,13 @@ password = [Custom_Spatial] -enabled = true +enabled = false country = Netherland region = town = Amsterdam [Custom_Temporal] -enabled = true +enabled = false businessDate = {{yesterday}} runDate = {{today}} diff --git a/source/golang/client/src/main.go b/source/golang/client/src/main.go index 9d53360..13a7d8b 100644 --- a/source/golang/client/src/main.go +++ b/source/golang/client/src/main.go @@ -17,6 +17,7 @@ type Parameters struct { task string sourcePath string recursive bool + update bool urlPrefix string configPath string definitionPath string @@ -99,6 +100,17 @@ func loadArguments(args []string) { } + + if args[ix] == "-R" || args[ix] == "--recursive" { + matched = true + params.recursive = true + } + + if args[ix] == "--update" { + matched = true + params.update = true + } + if args[ix] == "--title" { matched = true @@ -227,6 +239,11 @@ func main () { panic("Missing 'output' parameter") } + _, err := os.Stat(params.definitionPath) + if err == nil && params.update == false { + panic("MartiLQ document '"+ params.definitionPath+"' already exists and update not specified") + } + m := martilq.ProcessFilePath(params.configPath, params.sourcePath, params.recursive, params.urlPrefix, params.definitionPath ) if params.title != "" { m.Title = params.title diff --git a/source/golang/client/src/martilq/marti.go b/source/golang/client/src/martilq/marti.go index 66fbab5..bcd810a 100644 --- a/source/golang/client/src/martilq/marti.go +++ b/source/golang/client/src/martilq/marti.go @@ -210,6 +210,7 @@ func ProcessFilePath(ConfigPath string, SourcePath string, Recursive bool, UrlPr } fileStat, err := os.Stat(SourcePath) + fileAbs, err :=filepath.Abs(SourcePath) if err != nil { panic("Source path does not exist or is inaccessible: " + SourcePath) } else { @@ -222,24 +223,27 @@ func ProcessFilePath(ConfigPath string, SourcePath string, Recursive bool, UrlPr } if fileStat.IsDir() { + diffCheck := fileAbs+string(os.PathSeparator) - filepath.Walk(SourcePath, func(path string, info os.FileInfo, err error) error { + filepath.Walk(fileAbs, func(path string, info os.FileInfo, err error) error { if err != nil { log.Fatalf(err.Error()) } - if info.IsDir() { - if Recursive { - + if info.IsDir() == false { + diff := strings.Replace(path, diffCheck, "", -1) + if Recursive || diff == info.Name() { + url := UrlPrefix+strings.Replace(diff, "\\", "/", -1) + if UrlPrefix[0:6] == "file://" || UrlPrefix[0:1] == "\\\\" { + url = UrlPrefix+diff + } + m.AddResource(info.Name(), path, url) } - } else { - url := UrlPrefix+info.Name() - m.AddResource(info.Name(), path, url) } return nil }) } else { url := UrlPrefix+fileStat.Name() - m.AddResource(fileStat.Name(), SourcePath, url) + m.AddResource(fileStat.Name(), fileAbs, url) } }