diff --git a/.gitignore b/.gitignore index f6100fc..cc7bd53 100644 --- a/.gitignore +++ b/.gitignore @@ -11,9 +11,13 @@ buildNumber.properties .mvn/wrapper/maven-wrapper.jar test/powershell/results/ -docs/samples/powershell/test/ -docs/samples/python/test +docs/source/samples/powershell/test/ +docs/source/samples/python/test + +docs/build + source/python/client/__pycache__ -source/marti/source/golang/client/src/test +source/golang/client/src/test + diff --git a/README.md b/README.md index 95c8ca3..c055afb 100644 --- a/README.md +++ b/README.md @@ -40,7 +40,7 @@ information ## Transfer information The information in the **martiLQ** document is summarised below. For more detailed -information see [martiLQ definition](martiLQ.md) +information see [martiLQ definition](/docs/source/martiLQ.md) ### Mandatory information @@ -48,7 +48,7 @@ The mandatory information is: * Title * Unique identifier -* Distribution list - See Distribution section summary below or detailed document [Distribution](docs/distribution.md) +* Resource list - See Resource section summary below or detailed document [Resource](docs/source/resource.md) ### Optional information @@ -75,11 +75,11 @@ The optional information is: The information supplied can be extended by party agreement and there are place holders in the defintion. -### Distribution +### Resource -The distribution section is intended to allow multiple data files to be -grouped together. The distribution section can be repeated, but at least -one must be included. If the distribution is repeated it will commonly +The resource section is intended to allow multiple data files to be +grouped together. The resource section can be repeated, but at least +one must be included. If the resource is repeated it will commonly be for definiting multiple formats of the same data or batching of different data together from the same extract process. @@ -92,9 +92,9 @@ different data together from the same extract process. * Hash of document - The hash of the document, which can be blank especially for large documents * Hash algorithm -### Distribution optional +### Resource optional -The following are some of the optional items in the distribution section. See [Distribution](docs/distribution.md) +The following are some of the optional items in the resource section. See [Resource](docs/source/resources.md) for more details * Description diff --git a/docs/Makefile b/docs/Makefile new file mode 100644 index 0000000..d0c3cbf --- /dev/null +++ b/docs/Makefile @@ -0,0 +1,20 @@ +# Minimal makefile for Sphinx documentation +# + +# You can set these variables from the command line, and also +# from the environment for the first two. +SPHINXOPTS ?= +SPHINXBUILD ?= sphinx-build +SOURCEDIR = source +BUILDDIR = build + +# Put it first so that "make" without argument is like "make help". +help: + @$(SPHINXBUILD) -M help "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O) + +.PHONY: help Makefile + +# Catch-all target: route all unknown targets to Sphinx using the new +# "make mode" option. $(O) is meant as a shortcut for $(SPHINXOPTS). +%: Makefile + @$(SPHINXBUILD) -M $@ "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O) diff --git a/docs/ckan.md b/docs/ckan.md deleted file mode 100644 index 65e74df..0000000 --- a/docs/ckan.md +++ /dev/null @@ -1,5 +0,0 @@ - - -https://ckan.org/ - -Sample Json from https://data.gov.au/data/dataset/f2b7c2c1-f4ef-4ae9-aba5-45c19e4d3038 diff --git a/docs/ckan.txt b/docs/ckan.txt deleted file mode 100644 index 8b13789..0000000 --- a/docs/ckan.txt +++ /dev/null @@ -1 +0,0 @@ - diff --git a/docs/make.bat b/docs/make.bat new file mode 100644 index 0000000..061f32f --- /dev/null +++ b/docs/make.bat @@ -0,0 +1,35 @@ +@ECHO OFF + +pushd %~dp0 + +REM Command file for Sphinx documentation + +if "%SPHINXBUILD%" == "" ( + set SPHINXBUILD=sphinx-build +) +set SOURCEDIR=source +set BUILDDIR=build + +if "%1" == "" goto help + +%SPHINXBUILD% >NUL 2>NUL +if errorlevel 9009 ( + echo. + echo.The 'sphinx-build' command was not found. Make sure you have Sphinx + echo.installed, then set the SPHINXBUILD environment variable to point + echo.to the full path of the 'sphinx-build' executable. Alternatively you + echo.may add the Sphinx directory to PATH. + echo. + echo.If you don't have Sphinx installed, grab it from + echo.https://www.sphinx-doc.org/ + exit /b 1 +) + +%SPHINXBUILD% -M %1 %SOURCEDIR% %BUILDDIR% %SPHINXOPTS% %O% +goto end + +:help +%SPHINXBUILD% -M help %SOURCEDIR% %BUILDDIR% %SPHINXOPTS% %O% + +:end +popd diff --git a/docs/quality.md b/docs/quality.md deleted file mode 100644 index be75f4e..0000000 --- a/docs/quality.md +++ /dev/null @@ -1,60 +0,0 @@ -# Quality definition - -The **martiLQ** definition allows for the inclusion of a load quality -definition. This load quality definition is intended to be -able to be applied universally with common tools. As such not -all needs are covered. - -## Defined laod quality metrics - - -* Number of records in the document - This is the number of data primary records not the - count of end of lines and is agreed between parties. XML record counts could be based - on the number of primary segments under root. JSON records can be counted in a similar way. - The headers or trailling records are not counted - - -Sequence number - linked to the job producing the document and therefore a daily, weekly and monthly extracts for the -same document would have different sequence numbers - -Discourage us of magic formats for document names such as - -XT_PARTY_20210911_SQ00001_N000567891234_V01.DAT - -Header and Trailer records are not part of the quality definition except - -In fact the trailer record is intended to be replaced by this quality definition - -The header, if it exists, is only used where it identifies the column name sequence of the data. - -Effective and Process date ----------------- - -Row count - -Column count - -Depth count - - -Mandatory - data must be present and cannot be blank or null -Uniqueness - data value must be unique within the document -Data integrity - data exists within defined tolerances - - -row count 9999 - -column count 9999 - -column_name sum 9999 - -column_name gt 9999 -column_name lt 9999 -column_name eq 9999 -column_name eq " " -column_name ne 9999 -column_name ne " " == Check for value -column_name ge 9999 -column_name le " " -column_name in " ", " ", " " - -column_name is integer -column_name is decimal -column_name is unique - diff --git a/docs/README.md b/docs/source/README.md similarity index 93% rename from docs/README.md rename to docs/source/README.md index bebfe9e..f2d9163 100644 --- a/docs/README.md +++ b/docs/source/README.md @@ -21,4 +21,4 @@ can adjust if they resonate with your circumstances, 6. [Comparison of martiLQ definition](comparison.md) 7. [References](references.md) -[!INCLUDE [martiLQ High Level Definition](../martiLQ.md)] +[!INCLUDE [martiLQ High Level Definition](martiLQ.md)] diff --git a/docs/source/attributes.md b/docs/source/attributes.md new file mode 100644 index 0000000..b6e42f5 --- /dev/null +++ b/docs/source/attributes.md @@ -0,0 +1,32 @@ +# Attribute definition + +A Resource can list attributes related to the document / file. + +An attribute is a generic definition and conventions are +observed in the definitions that are captured here. + +## Attribute definition + +The Attribute consists of: + + * category - A value of "dataset", + * name + * function - A value such as "count" + * comparison - A comparisn value or NA. Values are "EQ", + "NE", "GT", "LT + * value - The value for the attribute based on the above complex key, excluding comparison + +A sample JSON is shown below which describes the +number of records in the file for the given format. + +```json + "attributes": [ + { + "category": "dataset", + "name": "records", + "function": "count", + "comparison": "EQ", + "value": "9" + } + ] +``` \ No newline at end of file diff --git a/docs/source/ckan.md b/docs/source/ckan.md new file mode 100644 index 0000000..e13a69e --- /dev/null +++ b/docs/source/ckan.md @@ -0,0 +1,15 @@ +# CKAN definition + +The **martiLQ** has used similar terms and structures that are found in the +CKAN API describing the resources. This similarity allows for simple mapping of values +from the CKAN format to **martiLQ** format. + +What **martiLQ** brings above the CKAN definition is attributes that can be +used to reconcile with the recieved data, plus the ability to define +compressed and encrypted resources. + +For more information on CKAN see https://ckan.org/ + +A sample Json to compare against the **martiLQ** definition +is https://data.gov.au/data/dataset/f2b7c2c1-f4ef-4ae9-aba5-45c19e4d3038 + diff --git a/docs/comparison.md b/docs/source/comparison.md similarity index 95% rename from docs/comparison.md rename to docs/source/comparison.md index a0d7cef..23872cf 100644 --- a/docs/comparison.md +++ b/docs/source/comparison.md @@ -34,10 +34,10 @@ Samples exist on CKAN integration. ## Magda and martiLQ Another source of data is [Magda](https://magda.io/) which has API metadata -definitions. Magda is more about data fedaration and as such provides +definitions. Magda is more about data federation and as such provides functionality on finding data sources and describing the contents. The Magda software is able to generate APIs and data content. This does not address the needs of data processing pipeline when reconciliation is required. -If you have Magda data sources then synergies exist between Magda and marti. +If you have Magda data sources then synergies exist between Magda and martiLQ. diff --git a/docs/source/conf.py b/docs/source/conf.py new file mode 100644 index 0000000..e52c17a --- /dev/null +++ b/docs/source/conf.py @@ -0,0 +1,64 @@ +# Configuration file for the Sphinx documentation builder. +# +# This file only contains a selection of the most common options. For a full +# list see the documentation: +# https://www.sphinx-doc.org/en/master/usage/configuration.html + +# -- Path setup -------------------------------------------------------------- + +# If extensions (or modules to document with autodoc) are in another directory, +# add these directories to sys.path here. If the directory is relative to the +# documentation root, use os.path.abspath to make it absolute, like shown here. +# +# import os +# import sys +# sys.path.insert(0, os.path.abspath('.')) + + +# -- Project information ----------------------------------------------------- + +project = 'MartiLQ' +copyright = '2021, meerkat@merebox.coom' +author = 'meerkat@merebox.coom' + +# The full version, including alpha/beta/rc tags +release = '0.0.1' + + +# -- General configuration --------------------------------------------------- + +# Add any Sphinx extension module names here, as strings. They can be +# extensions coming with Sphinx (named 'sphinx.ext.*') or your custom +# ones. +extensions = ['myst_parser'] + +# Add any paths that contain templates here, relative to this directory. +templates_path = ['_templates'] + +# List of patterns, relative to source directory, that match files and +# directories to ignore when looking for source files. +# This pattern also affects html_static_path and html_extra_path. +exclude_patterns = [ + 'samples/python/test/*', + 'samples/powershell/test/*', + ] + +source_suffix = { + '.rst': 'restructuredtext', + '.txt': 'restructuredtext', + '.md': 'markdown', +} + +# -- Options for HTML output ------------------------------------------------- + +# The theme to use for HTML and HTML Help pages. See the documentation for +# a list of builtin themes. +# +#html_theme = 'alabaster' +html_theme = 'sphinx_rtd_theme' + + +# Add any paths that contain custom static files (such as style sheets) here, +# relative to this directory. They are copied after the builtin static files, +# so a file named "default.css" will overwrite the builtin "default.css". +html_static_path = ['_static'] \ No newline at end of file diff --git a/docs/source/custom.md b/docs/source/custom.md new file mode 100644 index 0000000..bf32f49 --- /dev/null +++ b/docs/source/custom.md @@ -0,0 +1,9 @@ +# Custom Definition + +The custome definition section allows the inclusion of extensions +to the standard. To demonstrate the inclusion, there are three +sample extensions. These are: + + * Software - describing the **martiLQ** software version + * Spatial - Defining the geographical boundary of the documents + * Temporal - Defining date and time aspects of the documents diff --git a/docs/formats/csv.md b/docs/source/formats/csv.md similarity index 100% rename from docs/formats/csv.md rename to docs/source/formats/csv.md diff --git a/docs/formats/fixedlength.md b/docs/source/formats/fixedlength.md similarity index 100% rename from docs/formats/fixedlength.md rename to docs/source/formats/fixedlength.md diff --git a/docs/formats/sitpro.md b/docs/source/formats/json.md similarity index 100% rename from docs/formats/sitpro.md rename to docs/source/formats/json.md diff --git a/docs/formats/xml.md b/docs/source/formats/xml.md similarity index 100% rename from docs/formats/xml.md rename to docs/source/formats/xml.md diff --git a/docs/source/index.rst b/docs/source/index.rst new file mode 100644 index 0000000..8d1e280 --- /dev/null +++ b/docs/source/index.rst @@ -0,0 +1,37 @@ +.. MartiLQ documentation master file, created by + sphinx-quickstart on Sun Oct 31 14:30:06 2021. + You can adapt this file completely to your liking, but it should at least + contain the root `toctree` directive. + +Welcome to MartiLQ's documentation! +=================================== + +.. toctree:: + :hidden: + :maxdepth: 2 + + objective.md + martiLQ.md + why.md + what.md + when.md + who.md + resources.md + attributes.md + custom.md + quality.md + comparison.md + ckan.md + magda.md + references.md + +**martiLQ** stands for metadata reconcilation for transfer information, load quality. + +Before starting with **martiLQ** it is advisable to understand if it is right for +your organisation's needs. Information is available in a number of short +documents. Start with the (objectives)[objective.md] + +There are sample implementations which you +can adjust if they resonate with your circumstances. + +The source, documentation and samples are available at ``_ diff --git a/docs/magda.md b/docs/source/magda.md similarity index 79% rename from docs/magda.md rename to docs/source/magda.md index 569f65f..fd23e3f 100644 --- a/docs/magda.md +++ b/docs/source/magda.md @@ -1,4 +1,4 @@ - +# Magda definitions https://magda.io/ diff --git a/docs/source/martiLQ.md b/docs/source/martiLQ.md new file mode 100644 index 0000000..15d71b4 --- /dev/null +++ b/docs/source/martiLQ.md @@ -0,0 +1,75 @@ +# MartiLQ document + +The metadata reconciliation transfer information is referred +to as the **martiLQ** document throughout this documentation. + +The **martiLQ** document can be part of a message or a file +in its own right. The definition is curently a Json file. + + + +### Mandatory information + +The mandatory information is: + +* Title +* Unique identifier +* Resource list - See Resource section summary below or detailed document [Resource](resources.md) + +### Optional information + +The optional information is: + +* Description +* Modified +* Tags or keywords +* Publisher +* Contact point +* Acces level +* Rights +* Batch +* License +* Described By - A link to the metadata describing the document. + More detailed information could be supplied at the link +* Landing page +* Theme +* Custom list - List of custom entries, one being the **martiLQ** software details + see [custom](custom.md) + +### Information extension + +The information supplied can be extended by party agreement and there +are place holders in the defintion. + +### Resource + +The resource section is a list of documents or files that are to be grouped +together are listed under the same **martiLQ** definition. + +At least one document or file must be included. If the same resource is repeated +it will commonly be for definiting multiple formats, with each file having a +different extension. Commonly the definition includes at least the following +items: + +* Title - A summary description of the document contents +* Unique identifier - A unique identifier, commonly a GUID +* Document name - A name of the document such as thefile name +* Issued date - When the document was made available. The date can include time +* Modified - When the document was created or modified. This is the data and time +* Size of document - The document size in bytes +* URL - This can be ``file://``, ``https://``, ``ftp://``, etc resource location + +### Resource optional + +The following are some of the optional items in the resource section. See [Resource](resources.md) +for more details + +* Hash of document - The hash of the document, which can be blank especially for large documents +* Hash algorithm - Algoroithm used to generate the hash value or sign it +* Description - A more detailed description +* Version - A document version +* Encoding +* Content Type +* Compression +* Encryption +* Author diff --git a/docs/source/objective.md b/docs/source/objective.md new file mode 100644 index 0000000..9c08f31 --- /dev/null +++ b/docs/source/objective.md @@ -0,0 +1,22 @@ +# MartiLQ objective + +The objective of **martLQ** is to define a simle standard for +capturing the data files being transferred. It is not for +real time web service transactions. + +**martiLQ** is about file and document transfer and reconciling +that the all files have arrived and have not changed, and if so +required are also encrypted. + +The proposition is to have a common, machine readable format +for file exchange that: + + * ensures data load quality and reconciles + * can be used on Linux or Windows or Mac + * can be used with Python, Java, PowerShell, Golang, etc + * can be used by web services + * uses a text based format (JSON) + * can form part of the data processing pipeline + +And finally is easy to understand. + diff --git a/docs/source/quality.md b/docs/source/quality.md new file mode 100644 index 0000000..ffc1cc4 --- /dev/null +++ b/docs/source/quality.md @@ -0,0 +1,47 @@ +# Quality definition + +The **martiLQ** definition allows for the inclusion of load quality +definitions. The load quality definition is intended to be +able to be applied universally with common tools. Not +all needs are covered with the base definition but can be extended. + +## Defined load quality metrics + +* Sequential batch number - This is a decimal number defined at the **martiLQ** definition + and applies to all resources. The integer portion is for new batches and the fraction + part can be used for issues with the same data extract. such as requiring resend because + a resource was missing. + +* Number of records in the document - This is the number of data primary records not the + count of end of lines and is agreed between parties. XML record counts could be based + on the number of primary segments under root. JSON records can be counted in a similar way. + The headers or trailling records are not counted + +## Addresses deficiencies + +The **martiLQ** objective is to address deficiencies with alternative +data load quality approaches such as: + + * magic formats in file names + * identifying the number of files + * knowing when all files are ready + * separate documentation that is unlinked + * securing the data + * adding footers to the data, requiring custom file handlers + +## Extending load quality metrics + +**martLQ** framewokr and importantly is open to extension so that extra +load metrics appropriate to the situation can be included. + +### Extension ideas + +The following extensions for load quality can easily be included: + + * Mandatory data in column + * Uniqueness of data values + * Data values are within defined tolerances + * Check for data exclusions + +And all this information is included in the **martiLQ** definition +allowing for self describing load quality. diff --git a/docs/references.md b/docs/source/references.md similarity index 97% rename from docs/references.md rename to docs/source/references.md index 914cc53..7c585fc 100644 --- a/docs/references.md +++ b/docs/source/references.md @@ -1,4 +1,4 @@ - +# References https://dex.dss.gov.au/sites/default/files/documents/2021-06/data-exchange-protocols-june-2021.pdf diff --git a/docs/distribution.md b/docs/source/resources.md similarity index 92% rename from docs/distribution.md rename to docs/source/resources.md index d588bde..2219724 100644 --- a/docs/distribution.md +++ b/docs/source/resources.md @@ -1,6 +1,6 @@ -# Distribution definition +# Resources definition -The distrubution section defines the files that are grouped +The resources section defines the files that are grouped together by association. This association is not defined but can include different formats of the same data or a common batch extract such as end of day. @@ -10,7 +10,7 @@ compressed with a utility such as WinZIP or 7ZIP. In the situation where a ZIP file expands to multiple documents, then the expectation is that the ZIP file contains a **martiLQ** document describing its contents. -The elements in the distribution section are: +The elements in the resource section are: * Title * Document name - Commonly being absolute or relative file name. @@ -20,8 +20,9 @@ The elements in the distribution section are: * Size of file - The file size in bytes * Hash of file - The hash of the file, which can be blank especially for large files * Hash algorithm +* Attributes - List of attributes associated with the document -The following are optional in the distribution section. +The following are optional in the resource section. * Identifier * Description @@ -29,8 +30,7 @@ The following are optional in the distribution section. * Version - File version. The same file could be updated or this might denote the next version of a regular report. For example a daily extract will have the version number incremented every day and provide a new URL. The previous file can be retained. -* Format - if not specified then the consumer will in all likelihood use the file extension / mime type -* Media Type +* Content type - if not specified then the consumer will in all likelihood use the file extension / mime type * Expiry Date - The date and time that this file expires and can be removed from the download URL location. This is not the file retention period as might be required for archiving. * Described By - A link to the metadata describing this file data and format diff --git a/docs/samples/BSBDirectoryFtp.json b/docs/source/samples/BSBDirectoryFtp.json similarity index 100% rename from docs/samples/BSBDirectoryFtp.json rename to docs/source/samples/BSBDirectoryFtp.json diff --git a/docs/samples/BSBDirectoryHttp.json b/docs/source/samples/BSBDirectoryHttp.json similarity index 100% rename from docs/samples/BSBDirectoryHttp.json rename to docs/source/samples/BSBDirectoryHttp.json diff --git a/docs/samples/README.md b/docs/source/samples/README.md similarity index 100% rename from docs/samples/README.md rename to docs/source/samples/README.md diff --git a/docs/samples/asic_ckan_api.json b/docs/source/samples/asic_ckan_api.json similarity index 100% rename from docs/samples/asic_ckan_api.json rename to docs/source/samples/asic_ckan_api.json diff --git a/docs/samples/powershell/Invoke-SampleGenerateBsb.ps1 b/docs/source/samples/powershell/Invoke-SampleGenerateBsb.ps1 similarity index 100% rename from docs/samples/powershell/Invoke-SampleGenerateBsb.ps1 rename to docs/source/samples/powershell/Invoke-SampleGenerateBsb.ps1 diff --git a/docs/samples/powershell/Invoke-SampleGenerateBsbSecure.ps1 b/docs/source/samples/powershell/Invoke-SampleGenerateBsbSecure.ps1 similarity index 100% rename from docs/samples/powershell/Invoke-SampleGenerateBsbSecure.ps1 rename to docs/source/samples/powershell/Invoke-SampleGenerateBsbSecure.ps1 diff --git a/docs/samples/python/SampleFetchFtpBsb.py b/docs/source/samples/python/SampleFetchFtpBsb.py similarity index 100% rename from docs/samples/python/SampleFetchFtpBsb.py rename to docs/source/samples/python/SampleFetchFtpBsb.py diff --git a/docs/samples/python/SampleFetchHttpBsb.py b/docs/source/samples/python/SampleFetchHttpBsb.py similarity index 100% rename from docs/samples/python/SampleFetchHttpBsb.py rename to docs/source/samples/python/SampleFetchHttpBsb.py diff --git a/docs/samples/python/SampleGenerateBsb.py b/docs/source/samples/python/SampleGenerateBsb.py similarity index 100% rename from docs/samples/python/SampleGenerateBsb.py rename to docs/source/samples/python/SampleGenerateBsb.py diff --git a/docs/samples/python/SampleGenerateBsbSecure.py b/docs/source/samples/python/SampleGenerateBsbSecure.py similarity index 100% rename from docs/samples/python/SampleGenerateBsbSecure.py rename to docs/source/samples/python/SampleGenerateBsbSecure.py diff --git a/docs/what.md b/docs/source/what.md similarity index 80% rename from docs/what.md rename to docs/source/what.md index 1b175fc..1823d2b 100644 --- a/docs/what.md +++ b/docs/source/what.md @@ -1,6 +1,6 @@ # What is marti -The foundation pillar for the **martiLQ** framework is the [martiLQ document](../martiLQ.md) +The foundation pillar for the **martiLQ** framework is the [martiLQ document](martiLQ.md) that defines the reconciliation and other metadata of the document / file being transferred. A definition, while fundamental, benefits from having tools that can create, read and @@ -12,7 +12,7 @@ for various programming languages and situations. As many programming languages generate portable programs that can execute on multiple operating systems, the likelihood is that a tools exists for you. -The source for tools is provided in this repository and some have precompiled +The source for tools is provided in the Github repository and some have precompiled images. -See the [source](../source/) directory for more details. +See the project source directory for more details. diff --git a/docs/when.md b/docs/source/when.md similarity index 87% rename from docs/when.md rename to docs/source/when.md index cdac8ea..9e7f2ae 100644 --- a/docs/when.md +++ b/docs/source/when.md @@ -1,6 +1,7 @@ # When would you use martiLQ You are likely to start using the **martiLQ** framework when: + 1. you have no existing standard or framework or; 2. your existing standards no longer meets you needs or; 3. you are starting document exchange with another business or division that uses the framework or; @@ -8,7 +9,7 @@ You are likely to start using the **martiLQ** framework when: If you already have a standard and it works for you, and you have no upcoming (large) initiative that would benefit from the framework, then stick with what you have. The benefits -of the framework are unlikely to weigh in your framework +of the framework are unlikely to weigh in your framework. ## Read the material @@ -19,6 +20,3 @@ your direction will more than likely add to your support costs. **Note**: As the framework is based on open structure and can be processed independently without the tools, you can export the definitions to other tools / future methods. -See also: -1. [What the framework contains](what.md) -2. [Quality definition](quality.md) diff --git a/docs/who.md b/docs/source/who.md similarity index 84% rename from docs/who.md rename to docs/source/who.md index bc191d4..3ade812 100644 --- a/docs/who.md +++ b/docs/source/who.md @@ -1,8 +1,8 @@ -# Who is likely to use marti +# Who is likely to use martiLQ You are likely to find the **martiLQ** framework relevant if you: -1. Have many document exchanges +1. Have many document exchanges, such as End of Day batches 2. Need to verify or reconcile the documents ## Data exchanges @@ -33,8 +33,6 @@ in Mac OS X HFS. The Microsoft NTFS supports Alternate Data Streams to achieve Unfortunately this information is not transferrable to other systems. The proposition is to define a format for the sidecare file and provide common library tools that -can be be used on multiple platforms when exchanging documents / files. - -See also: -1. [When to use the framework](when.md) -2. [What the framework contains](what.md) +can be be used on multiple platforms when exchanging documents / files. Multiple documents can be +defined in a singel **martiLQ** definition which adds to efficiency and productivity if used +for End of Day or similar batches - or even single file transfers. diff --git a/docs/why.md b/docs/source/why.md similarity index 84% rename from docs/why.md rename to docs/source/why.md index 050554d..d6a9567 100644 --- a/docs/why.md +++ b/docs/source/why.md @@ -16,8 +16,9 @@ You would use **martiLQ** if any of the controls are a requirement for you. ## Documents Documents in this context are digital storage objects such as operating system files, -cloud storage objects or blobs. The document content has structure and contains multiple -records. +cloud storage objects or blobs. The document content can have structure and contains multiple +records or it can be unstructered such as PDFs. If you are including PDFs then they are +likely to be be supporting documents for your actual data files. The **martiLQ** framework is not intended to be used for single record transfers such as in single web transactions. It is for providing controls when moving large amounts of @@ -31,7 +32,3 @@ The framework does not replace your security, inflight encryption or encryption You are encouraged to use TLS or SSH to connect devices and transfer documents. Storage encryption and access controls for your documents is also relevant as part of the bigger picture. - -See also: -1. [Who would use the framework](who.md) -2. [When to use the framework](when.md) diff --git a/martiLQ.md b/martiLQ.md deleted file mode 100644 index c1e6230..0000000 --- a/martiLQ.md +++ /dev/null @@ -1,9 +0,0 @@ -# MartiLQ document - -The metadata reconciliation transfer information is referred -to as the **martiLQ** document throughout this documentation. - -The **martiLQ** document can be part of a message or a document -in its own right. If the document is a file then the recommended -name for the document is the same name as the data file, -including extension, with the added extension of ``.mti`` diff --git a/source/golang/client/src/config/batch.no b/source/golang/client/src/config/batch.no index 95170a3..baa5382 100644 --- a/source/golang/client/src/config/batch.no +++ b/source/golang/client/src/config/batch.no @@ -1 +1 @@ -0005.004 \ No newline at end of file +0001.001 \ No newline at end of file diff --git a/source/golang/client/src/config/martilq.ini b/source/golang/client/src/config/martilq.ini index 59bcec0..36ba8a7 100644 --- a/source/golang/client/src/config/martilq.ini +++ b/source/golang/client/src/config/martilq.ini @@ -2,7 +2,8 @@ [General] logPath = - +tempPath = +dataPath = [MartiLQ] @@ -18,12 +19,13 @@ theme = GOLANG [Resources] -author = -title = documentName +author = Mobidick +title = {{documentName}} state = active expires = 2:0:0 encoding = UTF-8 version = +urlPrefix = http://localhost/martilq/ [Hash] @@ -37,3 +39,17 @@ signKey_Password = proxy = username = password = + + +[Custom_Spatial] + +enabled = true +country = Netherland +region = +town = Amsterdam + +[Custom_Temporal] + +enabled = true +businessDate = {{yesterday}} +runDate = {{today}} diff --git a/source/golang/client/src/main.go b/source/golang/client/src/main.go index 1daa0c8..9d53360 100644 --- a/source/golang/client/src/main.go +++ b/source/golang/client/src/main.go @@ -11,10 +11,15 @@ import ( type Parameters struct { + + help bool + task string sourcePath string recursive bool + urlPrefix string configPath string + definitionPath string outputPath string title string @@ -37,6 +42,12 @@ func loadArguments(args []string) { for ix < maxArgs { matched := false + if args[ix] == "-h" || args[ix] == "--help" { + matched = true + params.help = true + break + } + if args[ix] == "-t" || args[ix] == "--task" { matched = true if ix < maxArgs { @@ -67,6 +78,16 @@ func loadArguments(args []string) { } } + if args[ix] == "-m" || args[ix] == "--martilq" { + matched = true + ix = ix + 1 + if ix < maxArgs { + params.definitionPath = args[ix] + } else { + panic("Missing parameter for MARTILQ") + } + } + if args[ix] == "-o" || args[ix] == "--output" { matched = true ix = ix + 1 @@ -126,64 +147,113 @@ func loadArguments(args []string) { } +func printHelp() { + + fmt.Println("") + fmt.Println("\t marticli_client ") + fmt.Println("\t =============== ") + fmt.Println("") + fmt.Println("\tThis program is intended as a simple reference implementation") + fmt.Println("\tin Go of the MartiLQ framework. It is does not provide all") + fmt.Println("\tthe possible functionality but enough to demonstrate the concept.") + fmt.Println("") + + fmt.Println(" The command line arguments are:") + fmt.Println("") + fmt.Println(" -h or --help : Display this help") + fmt.Println(" -t or --task : Execute a predefined task which are") + fmt.Println(" INIT initialise a new configuration file") + fmt.Println(" GEN generate a MartiLQ definition file") + fmt.Println(" RECON reconicile a MartiLQ definition file") + fmt.Println(" -c or --config : Configuration file used by all tasks") + fmt.Println(" This is the file written by the INIT task") + fmt.Println(" -s or --source : Source directory or file to build MartiLQ definition") + fmt.Println(" This is used by the GEN and RECON task") + fmt.Println(" -m or --martilq : MartiLQ definition file") + fmt.Println(" This is used by the GEN and RECON task") + fmt.Println(" The GEN task generates the file while") + fmt.Println(" RECON task reads the file") + fmt.Println(" -o or --output : Output file") + fmt.Println(" This is used by the RECON task") + + fmt.Println("") + fmt.Println(" --title : Title for the MartiLQ. Think of this as") + fmt.Println(" the job name") + fmt.Println(" This is used by the GEN task") + fmt.Println(" --description : Description for the MartiLQ. This can be text") + fmt.Println(" or a pointer to a file when the @ prefix is used") + fmt.Println(" This is used by the GEN task") + fmt.Println(" --landing : Landing page for the defintion in the MartiLQ") + fmt.Println(" This is best if it is a URL") + fmt.Println(" This is used by the GEN task") + + fmt.Println("") + +} + func main () { currentDirectory, _ := os.Getwd() params.sourcePath = currentDirectory - //params.outputPath = "" - //params.configPath = "" loadArguments(os.Args) matched := false - if params.task == "INIT" { - if params.configPath == "" { - panic("Missing 'config' parameter") - } - - c := martilq.NewConfiguration() - if c.SaveConfig(params.configPath) != true { - panic("Configuration not saved to: "+ params.configPath) - } - fmt.Println("Created MARTILQ INI definition: " + params.configPath) - matched = true - } - - if params.task == "GEN" { - - if params.sourcePath == "" { - panic("Missing 'source' parameter") - } - if params.outputPath == "" { - panic("Missing 'output' parameter") - } - - m := martilq.ProcessDirectory(params.configPath, params.sourcePath, params.recursive, params.outputPath ) - if params.title != "" { - m.Title = params.title - } - if params.landing != "" { - m.LandingPage = params.landing - } - if params.description != "" { - m.Description = params.description - } - m.Modified = time.Now() - m.Save(params.outputPath) - - fmt.Println("Created MARTILQ definition: " + params.outputPath) - matched = true - } - - if params.task == "RECON" { - - matched = true - } - - if matched { - fmt.Println("Completed " + params.task) + if params.help { + printHelp() } else { - fmt.Println("Unknown task: " + params.task) + + + if params.task == "INIT" { + if params.configPath == "" { + panic("Missing 'config' parameter") + } + + c := martilq.NewConfiguration() + if c.SaveConfig(params.configPath) != true { + panic("Configuration not saved to: "+ params.configPath) + } + fmt.Println("Created MARTILQ INI definition: " + params.configPath) + matched = true + } + + if params.task == "GEN" { + + if params.sourcePath == "" { + panic("Missing 'source' parameter") + } + if params.definitionPath == "" { + panic("Missing 'output' parameter") + } + + m := martilq.ProcessFilePath(params.configPath, params.sourcePath, params.recursive, params.urlPrefix, params.definitionPath ) + if params.title != "" { + m.Title = params.title + } + if params.landing != "" { + m.LandingPage = params.landing + } + if params.description != "" { + m.Description = params.description + } + m.Modified = time.Now() + m.Save(params.definitionPath) + + fmt.Println("Created MARTILQ definition: " + params.definitionPath) + matched = true + } + + if params.task == "RECON" { + + _ = martilq.ReconcileFilePath(params.configPath, params.sourcePath, params.recursive, params.definitionPath, params.outputPath ) + + matched = true + } + + if !matched { + printHelp() + } + } } diff --git a/source/golang/client/src/martilq/attributes_test.go b/source/golang/client/src/martilq/attributes_test.go index a6989d6..3c11964 100644 --- a/source/golang/client/src/martilq/attributes_test.go +++ b/source/golang/client/src/martilq/attributes_test.go @@ -4,7 +4,6 @@ import ( "testing" "strconv" "time" - "fmt" ) func TestAttr_Zip(t *testing.T) { @@ -38,7 +37,6 @@ func TestAttr_TemporalA(t *testing.T) { endDate := time.Now().AddDate(0,0,-1) a := NewDefaultTemporalAttributes(businessDate, runDate, false, startDate, endDate) - fmt.Println(a) if len(a) != 2 { t.Error("Arrays size not 2: " + strconv.Itoa(len(a))) } @@ -47,7 +45,6 @@ func TestAttr_TemporalA(t *testing.T) { } a = NewDefaultTemporalAttributes(businessDate, runDate, true, startDate, endDate) - fmt.Println(a) if len(a) != 4 { t.Error("Arrays size not 4: " + strconv.Itoa(len(a))) } diff --git a/source/golang/client/src/martilq/config.go b/source/golang/client/src/martilq/config.go index e89f0fa..8419133 100644 --- a/source/golang/client/src/martilq/config.go +++ b/source/golang/client/src/martilq/config.go @@ -20,6 +20,8 @@ type configuration struct { softwareName string logPath string + tempPath string + dataPath string publisher string contactPoint string @@ -33,6 +35,7 @@ type configuration struct { title string author string + urlPrefix string state string version string expires string @@ -50,6 +53,9 @@ type configuration struct { loaded bool configPath string + + temporal oTemporal + spatial oSpatial } @@ -63,7 +69,7 @@ func NewConfiguration() configuration { c.softwareName = GetSoftwareName() - c.title = "documentName" + c.title = "{{documentName}}" c.state = "active" c.accessLevel = "Confidential" c.rights = "Restricted" @@ -71,10 +77,14 @@ func NewConfiguration() configuration { c.encoding = cEncoding c.batchInc = 0.001 + c.urlPrefix = "file://" c.hash = true c.hashAlgorithm = "SHA256" c.loaded = false + c.spatial = GetSpatial() + c.temporal = GetTemporal() + configPath := findIni() if configPath != "" { c.LoadConfig(configPath) @@ -130,6 +140,8 @@ func (c *configuration) SaveConfig(ConfigPath string) bool { cfgini, _ := ini.LooseLoad("./martilq.ini") cfgini.Section("General").Key("logPath").SetValue (c.logPath) + cfgini.Section("General").Key("tempPath").SetValue (c.tempPath) + cfgini.Section("General").Key("dataPath").SetValue (c.dataPath) cfgini.Section("MartiLQ").Key("tags").SetValue(c.tags) cfgini.Section("MartiLQ").Key("publisher").SetValue(c.publisher) @@ -146,6 +158,7 @@ func (c *configuration) SaveConfig(ConfigPath string) bool { cfgini.Section("Resources").Key("expires").SetValue (c.expires) cfgini.Section("Resources").Key("encoding").SetValue (c.encoding) cfgini.Section("Resources").Key("version").SetValue (c.version) + cfgini.Section("Resources").Key("urlPrefix").SetValue (c.urlPrefix) cfgini.Section("Hash").Key("hashAlgorithm").SetValue (c.hashAlgorithm) cfgini.Section("Hash").Key("signKey_File").SetValue (c.signKey_File) @@ -161,6 +174,13 @@ func (c *configuration) SaveConfig(ConfigPath string) bool { WriteLog(fmt.Sprintf("Error saving to '%v'" , ConfigPath)) return false } + + res := c.spatial.SaveSpatial(ConfigPath) + res = c.temporal.SaveTemporal(ConfigPath) + if res { + + } + return true } @@ -191,6 +211,8 @@ func (c *configuration) LoadConfig(ConfigPath string) bool { } c.logPath = cfgini.Section("General").Key("logPath").MustString(c.logPath) + c.tempPath = cfgini.Section("General").Key("tempPath").MustString(c.tempPath) + c.dataPath = cfgini.Section("General").Key("dataPath").MustString(c.dataPath) c.tags = cfgini.Section("MartiLQ").Key("tags").MustString(c.tags) c.accessLevel = cfgini.Section("MartiLQ").Key("accessLevel").MustString(c.accessLevel) @@ -201,11 +223,12 @@ func (c *configuration) LoadConfig(ConfigPath string) bool { c.contactPoint = cfgini.Section("MartiLQ").Key("contactPoint").MustString(c.contactPoint) c.theme= cfgini.Section("MartiLQ").Key("theme").MustString(c.theme) - c.author = cfgini.Section("Resources").Key("title").MustString(c.title) + c.title = cfgini.Section("Resources").Key("title").MustString(c.title) c.author = cfgini.Section("Resources").Key("author").MustString(c.author) c.state = cfgini.Section("Resources").Key("state").MustString(c.state) c.expires = cfgini.Section("Resources").Key("expires").MustString(c.expires) c.encoding = cfgini.Section("Resources").Key("encoding").MustString(c.encoding) + c.urlPrefix = cfgini.Section("Resources").Key("urlPrefix").MustString(c.urlPrefix) c.hashAlgorithm = cfgini.Section("Hash").Key("hashAlgorithm").MustString(c.hashAlgorithm) c.signKey_File = cfgini.Section("Hash").Key("signKey_File").MustString(c.signKey_File) @@ -219,7 +242,9 @@ func (c *configuration) LoadConfig(ConfigPath string) bool { c.proxyUser = cfgini.Section("Network").Key("proxyUser").MustString(c.proxyUser) c.proxyCredential = cfgini.Section("Network").Key("proxyCredential").MustString(c.proxyCredential) - WriteLog(fmt.Sprintf("Loaded config file: %v", ConfigPath)) + c.spatial, _ = LoadSpatial(ConfigPath) + c.temporal, _ = LoadTemporal(ConfigPath) + c.configPath = ConfigPath } diff --git a/source/golang/client/src/martilq/custom.go b/source/golang/client/src/martilq/custom.go index bda24e1..974dd38 100644 --- a/source/golang/client/src/martilq/custom.go +++ b/source/golang/client/src/martilq/custom.go @@ -2,6 +2,8 @@ package martilq import ( "time" + "fmt" + "gopkg.in/ini.v1" ) type oSoftware struct { @@ -12,16 +14,18 @@ type oSoftware struct { } type oTemporal struct { + enabled bool Extension string `json:"extension"` - BusinessDate time.Time - RunDate time.Time + BusinessDate time.Time `json:"businessDate"` + RunDate time.Time `json:"runDate"` } type oSpatial struct { + enabled bool Extension string `json:"extension"` Country string `json:"country"` - Region string - Town string + Region string `json:"region"` + Town string `json:"town"` } func GetSoftware() oSoftware { @@ -48,3 +52,114 @@ func GetSpatial() oSpatial { return o } + + +func LoadSpatial(ConfigPath string) (oSpatial, error) { + o := oSpatial {} + o.Extension = "spatial" + + cfgini, err := ini.Load(ConfigPath) + if err != nil { + WriteLog(fmt.Sprintf("Fail to read file: %v", ConfigPath)) + fmt.Printf("Fail to read file: %v", err) + return o, err + } + + o.enabled = cfgini.Section("Custom_Spatial").Key("enabled").MustBool(o.enabled) + o.Country = cfgini.Section("Custom_Spatial").Key("country").MustString(o.Country) + o.Region = cfgini.Section("Custom_Spatial").Key("region").MustString(o.Region) + o.Town = cfgini.Section("Custom_Spatial").Key("town").MustString(o.Town) + + return o, nil +} + +func (s *oSpatial) SaveSpatial(ConfigPath string) bool { + + cfgini, _ := ini.Load(ConfigPath) + + cfgini.Section("Custom_Spatial").Key("enabled").SetValue("false") + cfgini.Section("Custom_Spatial").Key("country").SetValue(s.Country) + cfgini.Section("Custom_Spatial").Key("region").SetValue(s.Region) + cfgini.Section("Custom_Spatial").Key("town").SetValue(s.Town) + + err := cfgini.SaveTo(ConfigPath) + if err != nil { + WriteLog(fmt.Sprintf("Error saving to '%v'" , ConfigPath)) + return false + } + + return true + +} + + +func LoadTemporal(ConfigPath string) (oTemporal, error) { + o := oTemporal {} + o.Extension = "temporal" + + cfgini, err := ini.Load(ConfigPath) + if err != nil { + WriteLog(fmt.Sprintf("Fail to read file: %v", ConfigPath)) + fmt.Printf("Fail to read file: %v", err) + return o, err + } + + o.enabled = cfgini.Section("Custom_Temporal").Key("enabled").MustBool(o.enabled) + t := cfgini.Section("Custom_Temporal").Key("businessDate").MustString("") + if t != "" { + matched := false + if t == "{{now}}" { + matched = true + o.BusinessDate = time.Now() + } + if t == "{{yesterday}}" { + matched = true + o.BusinessDate = time.Now().AddDate(0,0,-1) + o.BusinessDate = time.Date(o.BusinessDate.Year(), o.BusinessDate.Month(), o.BusinessDate.Day(), 0, 0, 0, 0, time.Local) + } + if t == "{{today}}" { + matched = true + o.BusinessDate = time.Now() + o.BusinessDate = time.Date(o.BusinessDate.Year(), o.BusinessDate.Month(), o.BusinessDate.Day(), 0, 0, 0, 0, time.Local) + } + if !matched { + + } + } + t = cfgini.Section("Custom_Temporal").Key("runDate").MustString("") + if t != "" { + matched := false + if t == "{{now}}" { + matched = true + o.RunDate = time.Now() + } + if t == "{{today}}" { + matched = true + o.RunDate = time.Now() + o.RunDate = time.Date(o.RunDate.Year(), o.RunDate.Month(), o.RunDate.Day(), 0, 0, 0, 0, time.Local) + } + if !matched { + + } + } + + return o, nil +} + +func (s *oTemporal) SaveTemporal(ConfigPath string) bool { + + + cfgini, _ := ini.Load(ConfigPath) + + cfgini.Section("Custom_Temporal").Key("enabled").SetValue("false") + cfgini.Section("Custom_Temporal").Key("businessDate").SetValue("{{today}}") + cfgini.Section("Custom_Temporal").Key("runDate").SetValue("{{today}}") + + err := cfgini.SaveTo(ConfigPath) + if err != nil { + WriteLog(fmt.Sprintf("Error saving to '%v'" , ConfigPath)) + return false + } + + return true +} diff --git a/source/golang/client/src/martilq/marti.go b/source/golang/client/src/martilq/marti.go index c8d5a2a..66fbab5 100644 --- a/source/golang/client/src/martilq/marti.go +++ b/source/golang/client/src/martilq/marti.go @@ -55,10 +55,6 @@ func NewMarti() Marti { software := GetSoftware() m.Custom = append(m.Custom, software) - spatial := GetSpatial() - m.Custom = append(m.Custom, spatial) - temporal := GetTemporal() - m.Custom = append(m.Custom, temporal) m.config = NewConfiguration() @@ -103,6 +99,16 @@ func (m *Marti) LoadConfig(ConfigPath string) { if m.config.batch != "" { if m.config.batch[0] == '@' { _, err := os.Stat(m.config.batch[1:]) + if os.IsNotExist(err) { + // See if we can locate it in Config INI directory + _, fileb := filepath.Split(m.config.batch[1:]) + dirc, _ := filepath.Split(ConfigPath) + _, err := os.Stat(dirc + fileb) + if err == nil { + m.config.batch = "@" + dirc + fileb + } + } + _, err = os.Stat(m.config.batch[1:]) if os.IsNotExist(err) { WriteLog(fmt.Sprintf("Batch file '%v' does not exist" , m.config.batch)) } else { @@ -122,6 +128,15 @@ func (m *Marti) LoadConfig(ConfigPath string) { } } + if m.config.spatial.enabled == true { + spatial := m.config.spatial + m.Custom = append(m.Custom, spatial) + } + if m.config.temporal.enabled == true { + temporal := m.config.temporal + m.Custom = append(m.Custom, temporal) + } + } @@ -169,40 +184,74 @@ func (m *Marti) Save(pathFile string) bool { } -func ProcessDirectory(ConfigPath string, SourcePath string, Recursive bool, TargetPath string) Marti { +func ProcessFilePath(ConfigPath string, SourcePath string, Recursive bool, UrlPrefix string, DefinitionPath string) Marti { m := NewMarti() - _, err := os.Stat(TargetPath) - if err == nil { - m, err = Load(m.config, TargetPath) - if err != nil { - panic("Unable to load existing MartiLQ defintion: " + TargetPath) + if DefinitionPath != "" { + _, err := os.Stat(DefinitionPath) + if err == nil { + m, err = Load(m.config, DefinitionPath) + if err != nil { + panic("Unable to load existing MartiLQ definition: " + DefinitionPath) + } + // Update the batch number, minor version + m.Batch = math.Round((m.Batch + m.config.batchInc)/m.config.batchInc)*m.config.batchInc + m.config.LoadConfig(ConfigPath) + } else { + if ConfigPath != "" { + m.LoadConfig(ConfigPath) + } } - // Update the batch number, minor version - m.Batch = math.Round((m.Batch + m.config.batchInc)/m.config.batchInc)*m.config.batchInc - m.config.LoadConfig(ConfigPath) } else { if ConfigPath != "" { m.LoadConfig(ConfigPath) } } - filepath.Walk(SourcePath, func(path string, info os.FileInfo, err error) error { - if err != nil { - log.Fatalf(err.Error()) - } - if info.IsDir() { - if Recursive { + fileStat, err := os.Stat(SourcePath) + if err != nil { + panic("Source path does not exist or is inaccessible: " + SourcePath) + } else { - } - } else { - url := "file://"+info.Name() - m.AddResource(info.Name(), path, url) + if UrlPrefix == "" { + UrlPrefix = m.config.urlPrefix } - return nil - }) + if UrlPrefix == "" { + UrlPrefix = "file://" + } + + if fileStat.IsDir() { + + filepath.Walk(SourcePath, func(path string, info os.FileInfo, err error) error { + if err != nil { + log.Fatalf(err.Error()) + } + if info.IsDir() { + if Recursive { + + } + } else { + url := UrlPrefix+info.Name() + m.AddResource(info.Name(), path, url) + } + return nil + }) + } else { + url := UrlPrefix+fileStat.Name() + m.AddResource(fileStat.Name(), SourcePath, url) + } + } return m } + + +func ReconcileFilePath(ConfigPath string, SourcePath string, Recursive bool, DefinitionPath string, OutputPath string) Marti { + + m := NewMarti() + + + return m +} diff --git a/source/golang/client/src/martilq/marti_test.go b/source/golang/client/src/martilq/marti_test.go index 8e8bb68..94936f8 100644 --- a/source/golang/client/src/martilq/marti_test.go +++ b/source/golang/client/src/martilq/marti_test.go @@ -45,8 +45,8 @@ func TestMarti_DirectoryA(t *testing.T) { currentDirectory, _ := os.Getwd() SourcePath := currentDirectory Recursive := false - TargetPath := "../test/test_martilq_directoryA.json" - ProcessDirectory("", SourcePath, Recursive, TargetPath) + DefPath := "../test/test_martilq_directoryA.json" + ProcessFilePath("", SourcePath, Recursive, DefPath, "") } @@ -55,7 +55,7 @@ func TestMarti_DirectoryB(t *testing.T) { currentDirectory, _ := os.Getwd() SourcePath := currentDirectory Recursive := false - TargetPath := "../test/test_martilq_directoryB.json" - ProcessDirectory("../config/martilq.ini", SourcePath, Recursive, TargetPath) + DefPath := "../test/test_martilq_directoryB.json" + ProcessFilePath("../config/martilq.ini", SourcePath, Recursive, DefPath, "") } diff --git a/source/golang/client/src/martilq/resource.go b/source/golang/client/src/martilq/resource.go index e59bc7d..ac7274d 100644 --- a/source/golang/client/src/martilq/resource.go +++ b/source/golang/client/src/martilq/resource.go @@ -11,15 +11,15 @@ import ( type Resource struct { Title string `json:"title"` - Uid string `'json:"uid"` - DocumentName string `json:"documentName` + Uid string `json:"uid"` + DocumentName string `json:"documentName"` IssueDate time.Time `json:"issueDate"` Modified time.Time `json:"modified"` Expires time.Time `json:"expires"` State string `json:"state"` Author string `json:"author"` Length int64 `json:"length"` - Hash hash + Hash hash `json:"hash"` Description string `json:"description"` Url string `json:"url"` @@ -28,6 +28,7 @@ type Resource struct { Encoding string `json:"encoding"` Compression string `json:"compression"` Encryption string `json:"encryption"` + DescribedBy string `json:"describedBy"` Attributes []Attribute `json:"attributes"` } @@ -59,6 +60,11 @@ func NewMartiLQResource(config configuration, sourcePath string, urlPath string, } } + + if config.dataPath != "" { + + } + u := uuid.New() r.Uid = u.String() @@ -68,7 +74,7 @@ func NewMartiLQResource(config configuration, sourcePath string, urlPath string, r.Encoding = config.encoding r.DocumentName = stats.Name() - if config.title == "documentName" { + if config.title == "{{documentName}}" { r.Title = r.DocumentName } r.IssueDate = time.Now() diff --git a/source/golang/client/src/martilq_client.exe b/source/golang/client/src/martilq_client.exe new file mode 100644 index 0000000..8a8f514 Binary files /dev/null and b/source/golang/client/src/martilq_client.exe differ