"use strict"; const {Fragment: _Fragment, jsx: _jsx, jsxs: _jsxs} = arguments[0]; const {useMDXComponents: _provideComponents} = arguments[0]; function _createMdxContent(props) { const _components = { a: "a", blockquote: "blockquote", code: "code", div: "div", em: "em", h1: "h1", h2: "h2", h3: "h3", hr: "hr", iframe: "iframe", img: "img", li: "li", mermaid: "mermaid", ol: "ol", p: "p", path: "path", pre: "pre", span: "span", strong: "strong", svg: "svg", table: "table", tbody: "tbody", td: "td", th: "th", thead: "thead", tr: "tr", ul: "ul", ..._provideComponents(), ...props.components }; return _jsxs(_Fragment, { children: [_jsxs(_components.h1, { id: "notebook", children: [_jsx(_components.a, { className: "heading-link", href: "#notebook", children: _jsx(_components.svg, { xmlns: "http:www.w3.org/2000/svg", fill: "#ab2b65", viewBox: "0 0 20 20", className: "w-5 h-5", children: _jsx(_components.path, { fillRule: "evenodd", clipRule: "evenodd", d: "M9.493 2.853a.75.75 0 00-1.486-.205L7.545 6H4.198a.75.75 0 000 1.5h3.14l-.69 5H3.302a.75.75 0 000 1.5h3.14l-.435 3.148a.75.75 0 001.486.205L7.955 14h2.986l-.434 3.148a.75.75 0 001.486.205L12.456 14h3.346a.75.75 0 000-1.5h-3.14l.69-5h3.346a.75.75 0 000-1.5h-3.14l.435-3.147a.75.75 0 00-1.486-.205L12.045 6H9.059l.434-3.147zM8.852 7.5l-.69 5h2.986l.69-5H8.852z" }) }) }), "Notebook"] }), "\n", _jsx(_components.p, { children: "Our lab notebook. Informal thoughts. A very raw blog." }), "\n", _jsxs(_components.h1, { id: "data-literate---a-small-product-idea-2021-05-17-rufuspollock", children: [_jsx(_components.a, { className: "heading-link", href: "#data-literate---a-small-product-idea-2021-05-17-rufuspollock", children: _jsx(_components.svg, { xmlns: "http:www.w3.org/2000/svg", fill: "#ab2b65", viewBox: "0 0 20 20", className: "w-5 h-5", children: _jsx(_components.path, { fillRule: "evenodd", clipRule: "evenodd", d: "M9.493 2.853a.75.75 0 00-1.486-.205L7.545 6H4.198a.75.75 0 000 1.5h3.14l-.69 5H3.302a.75.75 0 000 1.5h3.14l-.435 3.148a.75.75 0 001.486.205L7.955 14h2.986l-.434 3.148a.75.75 0 001.486.205L12.456 14h3.346a.75.75 0 000-1.5h-3.14l.69-5h3.346a.75.75 0 000-1.5h-3.14l.435-3.147a.75.75 0 00-1.486-.205L12.045 6H9.059l.434-3.147zM8.852 7.5l-.69 5h2.986l.69-5H8.852z" }) }) }), "Data Literate - a small Product Idea 2021-05-17 @rufuspollock"] }), "\n", _jsx(_components.p, { children: "I want to write a README with data and vis in it and preview it …" }), "\n", _jsxs(_components.ul, { children: ["\n", _jsxs(_components.li, { children: ["Markdown is becoming a lingua franca for writing developer and even research docs", "\n", _jsxs(_components.ul, { children: ["\n", _jsx(_components.li, { children: "It's quick and ascii-like" }), "\n", _jsx(_components.li, { children: "It's widely supported" }), "\n", _jsx(_components.li, { children: "It's extensible …" }), "\n"] }), "\n"] }), "\n", _jsxs(_components.li, { children: ["Frontend tooling is rapidly evolving …", "\n", _jsxs(_components.ul, { children: ["\n", _jsx(_components.li, { children: "The distant between code and a tool is declining => I might as well write code …" }), "\n"] }), "\n"] }), "\n", _jsx(_components.li, { children: "MDX = Markdown + react" }), "\n", _jsx(_components.li, { children: "RStudio did this a while ago …" }), "\n", _jsxs(_components.li, { children: ["Missing part is data …", "\n", _jsxs(_components.ul, { children: ["\n", _jsx(_components.li, { children: "You have juputer notebooks etc … => they are quite high end …" }), "\n"] }), "\n"] }), "\n"] }), "\n", _jsx(_components.pre, { children: _jsxs(_components.code, { className: "language-auto code-highlight", children: [_jsx(_components.span, { className: "code-line", children: "Notebooks (jupyter, literate programming) ==> \n" }), _jsx(_components.span, { className: "code-line", children: " Write text and code together\n" }), _jsx(_components.span, { className: "code-line", children: " Write code like in a terminal\n" }), _jsx(_components.span, { className: "code-line", children: " Data oriented\n" })] }) }), "\n", _jsx(_components.p, { children: "Visualization" }), "\n", _jsx(_components.p, { children: "React" }), "\n", _jsx(_components.p, { children: "Markdown …" }), "\n", _jsx(_components.hr, {}), "\n", _jsx(_components.p, { children: "Here the kinds of doc i want to write" }), "\n", _jsx(_components.pre, { children: _jsxs(_components.code, { className: "language-auto code-highlight", children: [_jsx(_components.span, { className: "code-line", children: "## A Dataset\n" }), _jsx(_components.span, { className: "code-line", children: "\n" }), _jsx(_components.span, { className: "code-line", children: "\\```\n" }), _jsx(_components.span, { className: "code-line", children: "# Global Solar Supply (Annual)\n" }), _jsx(_components.span, { className: "code-line", children: "\n" }), _jsx(_components.span, { className: "code-line", children: "Solar energy supply globally.\n" }), _jsx(_components.span, { className: "code-line", children: "\n" }), _jsx(_components.span, { className: "code-line", children: "Source: International Energy Association https://www.iea.org/reports/solar-pv.\n" }), _jsx(_components.span, { className: "code-line", children: "\n" }), _jsx(_components.span, { className: "code-line", children: "| Year | Generation (TWh) | % of total energy |\n" }), _jsx(_components.span, { className: "code-line", children: "|--|--|\n" }), _jsx(_components.span, { className: "code-line", children: "|2008|12|\n" }), _jsx(_components.span, { className: "code-line", children: "|2009|20|\n" }), _jsx(_components.span, { className: "code-line", children: "|2010|32|\n" }), _jsx(_components.span, { className: "code-line", children: "|2011|63|\n" }), _jsx(_components.span, { className: "code-line", children: "|2012|99|\n" }), _jsx(_components.span, { className: "code-line", children: "|2013|139|\n" }), _jsx(_components.span, { className: "code-line", children: "|2014|190|\n" }), _jsx(_components.span, { className: "code-line", children: "|2015|251|\n" }), _jsx(_components.span, { className: "code-line", children: "|2016|329|\n" }), _jsx(_components.span, { className: "code-line", children: "|2017|444|\n" }), _jsx(_components.span, { className: "code-line", children: "|2018|585|\n" }), _jsx(_components.span, { className: "code-line", children: "|2019|720| 2.7 |\n" }), _jsx(_components.span, { className: "code-line", children: "\\```\n" }), _jsx(_components.span, { className: "code-line", children: "\n" }), _jsx(_components.span, { className: "code-line", children: "Europe Brent Spot Prices (Annual/ Monthly/ Weekly/ Daily) from U.S. Energy Information Administration (EIA).\n" }), _jsx(_components.span, { className: "code-line", children: "\n" }), _jsx(_components.span, { className: "code-line", children: "Source: https://www.eia.gov/dnav/pet/hist/RBRTEd.htm\n" })] }) }), "\n", _jsxs(_components.h3, { id: "notes", children: [_jsx(_components.a, { className: "heading-link", href: "#notes", children: _jsx(_components.svg, { xmlns: "http:www.w3.org/2000/svg", fill: "#ab2b65", viewBox: "0 0 20 20", className: "w-5 h-5", children: _jsx(_components.path, { fillRule: "evenodd", clipRule: "evenodd", d: "M9.493 2.853a.75.75 0 00-1.486-.205L7.545 6H4.198a.75.75 0 000 1.5h3.14l-.69 5H3.302a.75.75 0 000 1.5h3.14l-.435 3.148a.75.75 0 001.486.205L7.955 14h2.986l-.434 3.148a.75.75 0 001.486.205L12.456 14h3.346a.75.75 0 000-1.5h-3.14l.69-5h3.346a.75.75 0 000-1.5h-3.14l.435-3.147a.75.75 0 00-1.486-.205L12.045 6H9.059l.434-3.147zM8.852 7.5l-.69 5h2.986l.69-5H8.852z" }) }) }), "Notes"] }), "\n", _jsxs(_components.p, { children: ["R Markdown - ", _jsx(_components.a, { href: "https://rmarkdown.rstudio.com", target: "_blank", rel: "noopener noreferrer", children: "https://rmarkdown.rstudio.com" })] }), "\n", _jsxs(_components.blockquote, { children: ["\n", _jsx(_components.p, { children: "Use a productive notebook interface to weave together narrative text and code to produce elegantly formatted output." }), "\n"] }), "\n", _jsxs(_components.h2, { id: "a-dms-is-a-tool-a-data-portal-is-a-solution-2021-03-14-rufuspollock", children: [_jsx(_components.a, { className: "heading-link", href: "#a-dms-is-a-tool-a-data-portal-is-a-solution-2021-03-14-rufuspollock", children: _jsx(_components.svg, { xmlns: "http:www.w3.org/2000/svg", fill: "#ab2b65", viewBox: "0 0 20 20", className: "w-5 h-5", children: _jsx(_components.path, { fillRule: "evenodd", clipRule: "evenodd", d: "M9.493 2.853a.75.75 0 00-1.486-.205L7.545 6H4.198a.75.75 0 000 1.5h3.14l-.69 5H3.302a.75.75 0 000 1.5h3.14l-.435 3.148a.75.75 0 001.486.205L7.955 14h2.986l-.434 3.148a.75.75 0 001.486.205L12.456 14h3.346a.75.75 0 000-1.5h-3.14l.69-5h3.346a.75.75 0 000-1.5h-3.14l.435-3.147a.75.75 0 00-1.486-.205L12.045 6H9.059l.434-3.147zM8.852 7.5l-.69 5h2.986l.69-5H8.852z" }) }) }), "A DMS is a tool, a Data Portal is a solution 2021-03-14 @rufuspollock"] }), "\n", _jsx(_components.p, { children: "Over the years, we've seen many different terms used to describe software like CKAN and the solutions it is used to create e.g. data catalog, data portal, data management system, data platform etc." }), "\n", _jsxs(_components.p, { children: ["Over time, personally, I've converged towards ", _jsx(_components.a, { href: "/dms/dms", children: "data management system (DMS)" }), " and ", _jsx(_components.a, { href: "/docs/dms/data-portals", children: "data portal" }), ". But I've still got two terms and even people in my own team ask me to clarify what the difference is. Recently it became clear to me:"] }), "\n", _jsx(_components.p, { children: _jsx(_components.strong, { children: "A data management system (DMS) is a tool. A data portal is a solution." }) }), "\n", _jsx(_components.p, { children: "And a data management system is a tool you can use to build a data portal. Just like you can use a hammer to build a house." }), "\n", _jsxs(_components.h2, { id: "data-factory-concepts-2020-09-03-rufuspollock", children: [_jsx(_components.a, { className: "heading-link", href: "#data-factory-concepts-2020-09-03-rufuspollock", children: _jsx(_components.svg, { xmlns: "http:www.w3.org/2000/svg", fill: "#ab2b65", viewBox: "0 0 20 20", className: "w-5 h-5", children: _jsx(_components.path, { fillRule: "evenodd", clipRule: "evenodd", d: "M9.493 2.853a.75.75 0 00-1.486-.205L7.545 6H4.198a.75.75 0 000 1.5h3.14l-.69 5H3.302a.75.75 0 000 1.5h3.14l-.435 3.148a.75.75 0 001.486.205L7.955 14h2.986l-.434 3.148a.75.75 0 001.486.205L12.456 14h3.346a.75.75 0 000-1.5h-3.14l.69-5h3.346a.75.75 0 000-1.5h-3.14l.435-3.147a.75.75 0 00-1.486-.205L12.045 6H9.059l.434-3.147zM8.852 7.5l-.69 5h2.986l.69-5H8.852z" }) }) }), "Data Factory Concepts 2020-09-03 @rufuspollock"] }), "\n", _jsx(_components.p, { children: "Had this conceptual diagram hanging round for a couple of years." }), "\n", _jsx(_components.pre, { children: _jsxs(_components.code, { className: "language-auto code-highlight", children: [_jsx(_components.span, { className: "code-line", children: "Objects\n" }), _jsx(_components.span, { className: "code-line", children: "\n" }), _jsx(_components.span, { className: "code-line", children: "Row\n" }), _jsx(_components.span, { className: "code-line", children: " File\n" }), _jsx(_components.span, { className: "code-line", children: " Dataset\n" }), _jsx(_components.span, { className: "code-line", children: "\n" }), _jsx(_components.span, { className: "code-line", children: "Transformations\n" }), _jsx(_components.span, { className: "code-line", children: "\n" }), _jsx(_components.span, { className: "code-line", children: "Operator\n" }), _jsx(_components.span, { className: "code-line", children: " Pipeline\n" }), _jsx(_components.span, { className: "code-line", children: " Flow \n" })] }) }), "\n", _jsx(_components.p, { children: "NTS" }), "\n", _jsxs(_components.ul, { children: ["\n", _jsx(_components.li, { children: "A factory could be a (DA)G of flows b/c could be dependencies between them … e.g. run ComputeKeyMetrics only after all other flows have run …" }), "\n", _jsx(_components.li, { children: "But not always like that: flows can be completely independent." }), "\n"] }), "\n", _jsxs(_components.h2, { id: "current-data-factory-components-early-2019", children: [_jsx(_components.a, { className: "heading-link", href: "#current-data-factory-components-early-2019", children: _jsx(_components.svg, { xmlns: "http:www.w3.org/2000/svg", fill: "#ab2b65", viewBox: "0 0 20 20", className: "w-5 h-5", children: _jsx(_components.path, { fillRule: "evenodd", clipRule: "evenodd", d: "M9.493 2.853a.75.75 0 00-1.486-.205L7.545 6H4.198a.75.75 0 000 1.5h3.14l-.69 5H3.302a.75.75 0 000 1.5h3.14l-.435 3.148a.75.75 0 001.486.205L7.955 14h2.986l-.434 3.148a.75.75 0 001.486.205L12.456 14h3.346a.75.75 0 000-1.5h-3.14l.69-5h3.346a.75.75 0 000-1.5h-3.14l.435-3.147a.75.75 0 00-1.486-.205L12.045 6H9.059l.434-3.147zM8.852 7.5l-.69 5h2.986l.69-5H8.852z" }) }) }), "Current Data Factory Components (early 2019)"] }), "\n", _jsx(_components.pre, { children: _jsxs(_components.code, { className: "language-auto code-highlight", children: [_jsx(_components.span, { className: "code-line", children: " Factory - Runners, SaaS platform etc\n" }), _jsx(_components.span, { className: "code-line", children: "\n" }), _jsx(_components.span, { className: "code-line", children: "datapackage-pipelines -> (dataflows-server / dataflows-runner)\n" }), _jsx(_components.span, { className: "code-line", children: "dataflows-cli : generators, hello-world, 'init', 'run'\n" }), _jsx(_components.span, { className: "code-line", children: "goodtables.io\n" }), _jsx(_components.span, { className: "code-line", children: "\"blueprints\": standard setups for a factory (auto-guessed?)\n" }), _jsx(_components.span, { className: "code-line", children: "\n" }), _jsx(_components.span, { className: "code-line", children: " DataFlows: Flow Libs\n" }), _jsx(_components.span, { className: "code-line", children: "\n" }), _jsx(_components.span, { className: "code-line", children: "dataflows : processor definition and chaining\n" }), _jsx(_components.span, { className: "code-line", children: "processors-library: stdlib, user contributed items [dataflows-load-csv]\n" }), _jsx(_components.span, { className: "code-line", children: "\n" }), _jsx(_components.span, { className: "code-line", children: " Data Package Libs\n" }), _jsx(_components.span, { className: "code-line", children: "\n" }), _jsx(_components.span, { className: "code-line", children: "data.py, DataPackage-py, GoodTables, ...\n" }), _jsx(_components.span, { className: "code-line", children: "tabulator, tableschema\n" })] }) }), "\n", _jsxs(_components.h2, { id: "composition-vs-inheritance-approaches-to-building-applications-and-esp-web-apps-2020-08-20-rufuspollock", children: [_jsx(_components.a, { className: "heading-link", href: "#composition-vs-inheritance-approaches-to-building-applications-and-esp-web-apps-2020-08-20-rufuspollock", children: _jsx(_components.svg, { xmlns: "http:www.w3.org/2000/svg", fill: "#ab2b65", viewBox: "0 0 20 20", className: "w-5 h-5", children: _jsx(_components.path, { fillRule: "evenodd", clipRule: "evenodd", d: "M9.493 2.853a.75.75 0 00-1.486-.205L7.545 6H4.198a.75.75 0 000 1.5h3.14l-.69 5H3.302a.75.75 0 000 1.5h3.14l-.435 3.148a.75.75 0 001.486.205L7.955 14h2.986l-.434 3.148a.75.75 0 001.486.205L12.456 14h3.346a.75.75 0 000-1.5h-3.14l.69-5h3.346a.75.75 0 000-1.5h-3.14l.435-3.147a.75.75 0 00-1.486-.205L12.045 6H9.059l.434-3.147zM8.852 7.5l-.69 5h2.986l.69-5H8.852z" }) }) }), "Composition vs Inheritance approaches to building applications and esp web apps 2020-08-20 @rufuspollock"] }), "\n", _jsx(_components.p, { children: "tl;dr: composition is better than inheritance but many systems are built with inheritance" }), "\n", _jsx(_components.p, { children: "Imagine we want a page like this:" }), "\n", _jsx(_components.pre, { children: _jsxs(_components.code, { className: "language-auto code-highlight", children: [_jsx(_components.span, { className: "code-line", children: "

\n" }), _jsx(_components.span, { className: "code-line", children: "{{title}}\n" }), _jsx(_components.span, { className: "code-line", children: "\n" })] }) }), "\n", _jsx(_components.p, { children: "Inheritance / Slot model" }), "\n", _jsx(_components.pre, { children: _jsxs(_components.code, { className: "language-auto code-highlight", children: [_jsx(_components.span, { className: "code-line", children: "def render_home:\n" }), _jsx(_components.span, { className: "code-line", children: " return render('base_template.html'< {\n" }), _jsx(_components.span, { className: "code-line", children: " title=\"hello world\"\n" }), _jsx(_components.span, { className: "code-line", children: " })\n" })] }) }), "\n", _jsx(_components.p, { children: "Composition / declarative" }), "\n", _jsx(_components.pre, { children: _jsxs(_components.code, { className: "language-auto code-highlight", children: [_jsx(_components.span, { className: "code-line", children: "def render_home:\n" }), _jsx(_components.span, { className: "code-line", children: " mytitle = 'hello world'\n" }), _jsx(_components.span, { className: "code-line", children: " response.write(get_header())\n" }), _jsx(_components.span, { className: "code-line", children: " response.write(mytitle)\n" }), _jsx(_components.span, { className: "code-line", children: " response.write(get_footer()\n" })] }) }), "\n", _jsx(_components.p, { children: "You can write templates two ways:" }), "\n", _jsxs(_components.h3, { id: "inheritance", children: [_jsx(_components.a, { className: "heading-link", href: "#inheritance", children: _jsx(_components.svg, { xmlns: "http:www.w3.org/2000/svg", fill: "#ab2b65", viewBox: "0 0 20 20", className: "w-5 h-5", children: _jsx(_components.path, { fillRule: "evenodd", clipRule: "evenodd", d: "M9.493 2.853a.75.75 0 00-1.486-.205L7.545 6H4.198a.75.75 0 000 1.5h3.14l-.69 5H3.302a.75.75 0 000 1.5h3.14l-.435 3.148a.75.75 0 001.486.205L7.955 14h2.986l-.434 3.148a.75.75 0 001.486.205L12.456 14h3.346a.75.75 0 000-1.5h-3.14l.69-5h3.346a.75.75 0 000-1.5h-3.14l.435-3.147a.75.75 0 00-1.486-.205L12.045 6H9.059l.434-3.147zM8.852 7.5l-.69 5h2.986l.69-5H8.852z" }) }) }), "Inheritance"] }), "\n", _jsx(_components.p, { children: "Base template" }), "\n", _jsx(_components.pre, { className: "language-html", children: _jsxs(_components.code, { className: "language-html code-highlight", children: [_jsx(_components.span, { className: "code-line", children: "# base.html\n" }), _jsxs(_components.span, { className: "code-line", children: [_jsxs(_components.span, { className: "token tag", children: [_jsxs(_components.span, { className: "token tag", children: [_jsx(_components.span, { className: "token punctuation", children: "<" }), "header"] }), _jsx(_components.span, { className: "token punctuation", children: ">" })] }), "\n"] }), _jsxs(_components.span, { className: "code-line", children: [_jsxs(_components.span, { className: "token tag", children: [_jsxs(_components.span, { className: "token tag", children: [_jsx(_components.span, { className: "token punctuation", children: "<" }), "title"] }), _jsx(_components.span, { className: "token punctuation", children: ">" })] }), "{{title}}", _jsxs(_components.span, { className: "token tag", children: [_jsxs(_components.span, { className: "token tag", children: [_jsx(_components.span, { className: "token punctuation", children: "" })] }), "\n"] }), _jsx(_components.span, { className: "code-line", children: "\n" }), _jsx(_components.span, { className: "code-line", children: "{{content}}\n" }), _jsxs(_components.span, { className: "code-line", children: [_jsxs(_components.span, { className: "token tag", children: [_jsxs(_components.span, { className: "token tag", children: [_jsx(_components.span, { className: "token punctuation", children: "" })] }), "\n"] })] }) }), "\n", _jsx(_components.p, { children: _jsx(_components.code, { className: "language-auto", children: "blog-post.html" }) }), "\n", _jsx(_components.pre, { children: _jsxs(_components.code, { className: "language-auto code-highlight", children: [_jsx(_components.span, { className: "code-line", children: "{%extends base.html %}\n" }), _jsx(_components.span, { className: "code-line", children: "{{title}} -- Blog\n" })] }) }), "\n", _jsxs(_components.h3, { id: "composition", children: [_jsx(_components.a, { className: "heading-link", href: "#composition", children: _jsx(_components.svg, { xmlns: "http:www.w3.org/2000/svg", fill: "#ab2b65", viewBox: "0 0 20 20", className: "w-5 h-5", children: _jsx(_components.path, { fillRule: "evenodd", clipRule: "evenodd", d: "M9.493 2.853a.75.75 0 00-1.486-.205L7.545 6H4.198a.75.75 0 000 1.5h3.14l-.69 5H3.302a.75.75 0 000 1.5h3.14l-.435 3.148a.75.75 0 001.486.205L7.955 14h2.986l-.434 3.148a.75.75 0 001.486.205L12.456 14h3.346a.75.75 0 000-1.5h-3.14l.69-5h3.346a.75.75 0 000-1.5h-3.14l.435-3.147a.75.75 0 00-1.486-.205L12.045 6H9.059l.434-3.147zM8.852 7.5l-.69 5h2.986l.69-5H8.852z" }) }) }), "Composition"] }), "\n", _jsx(_components.p, { children: _jsx(_components.code, { className: "language-auto", children: "blog-post.html" }) }), "\n", _jsx(_components.pre, { children: _jsxs(_components.code, { className: "language-auto code-highlight", children: [_jsx(_components.span, { className: "code-line", children: "\n" }), _jsx(_components.span, { className: "code-line", children: "\n" }), _jsx(_components.span, { className: "code-line", children: "{{content}}\n" }), _jsx(_components.span, { className: "code-line", children: "\n" }), _jsx(_components.span, { className: "code-line", children: "\n" })] }) }), "\n", _jsxs(_components.h2, { id: "how-views-should-work-in-ckan-v3-next-gen-2020-08-10--rufuspollock", children: [_jsx(_components.a, { className: "heading-link", href: "#how-views-should-work-in-ckan-v3-next-gen-2020-08-10--rufuspollock", children: _jsx(_components.svg, { xmlns: "http:www.w3.org/2000/svg", fill: "#ab2b65", viewBox: "0 0 20 20", className: "w-5 h-5", children: _jsx(_components.path, { fillRule: "evenodd", clipRule: "evenodd", d: "M9.493 2.853a.75.75 0 00-1.486-.205L7.545 6H4.198a.75.75 0 000 1.5h3.14l-.69 5H3.302a.75.75 0 000 1.5h3.14l-.435 3.148a.75.75 0 001.486.205L7.955 14h2.986l-.434 3.148a.75.75 0 001.486.205L12.456 14h3.346a.75.75 0 000-1.5h-3.14l.69-5h3.346a.75.75 0 000-1.5h-3.14l.435-3.147a.75.75 0 00-1.486-.205L12.045 6H9.059l.434-3.147zM8.852 7.5l-.69 5h2.986l.69-5H8.852z" }) }) }), "How Views should work in CKAN v3 (Next Gen) 2020-08-10 @rufuspollock"] }), "\n", _jsx(_components.p, { children: "Two key points:" }), "\n", _jsxs(_components.ol, { children: ["\n", _jsx(_components.li, { children: "Views should become an explicit (data) project level object" }), "\n", _jsxs(_components.li, { children: ["Previews: should be very simple and work ", _jsx(_components.em, { children: "without" }), " data API"] }), "\n"] }), "\n", _jsx(_components.p, { children: "Why?" }), "\n", _jsxs(_components.ul, { children: ["\n", _jsxs(_components.li, { children: ["Views should become an explicit (data) project level object", "\n", _jsxs(_components.ul, { children: ["\n", _jsx(_components.li, { children: "So I can show a view on dataset page (atm I can't)" }), "\n", _jsx(_components.li, { children: "I have multiview view inside reclinejs but rest are single views … (this is confusing)" }), "\n", _jsx(_components.li, { children: "I can't create views across multiple resources" }), "\n", _jsx(_components.li, { children: "They are nested under resources but they aren't really part of a resource" }), "\n"] }), "\n"] }), "\n", _jsxs(_components.li, { children: ["Previews: should be very simple and work ", _jsx(_components.em, { children: "without" }), " data API", "\n", _jsxs(_components.ul, { children: ["\n", _jsx(_components.li, { children: "so they work with revisions (atm views often depend on data API which causes problems with revisions and viewing old revisions of resources)" }), "\n"] }), "\n"] }), "\n"] }), "\n", _jsx(_components.p, { children: "Distinguish 3 concepts" }), "\n", _jsxs(_components.ul, { children: ["\n", _jsxs(_components.li, { children: ["Preview: a very simple method for previewing specific raw data types e.g. csv, excel, json, xml, text, geojson etc …", "\n", _jsxs(_components.ul, { children: ["\n", _jsx(_components.li, { children: "Key aspect are ability to sample a part and to present." }), "\n"] }), "\n"] }), "\n", _jsx(_components.li, { children: "Viz: graph, map, … (visualizations)" }), "\n", _jsx(_components.li, { children: "Query UI: UI for creating queries" }), "\n", _jsx(_components.li, { children: "Viz Builder: a UI for creating charts, tables, maps etc" }), "\n", _jsx(_components.li, { children: "Explorer (dashboard): combines query UI, Viz Builder and Viz Renderer" }), "\n"] }), "\n", _jsxs(_components.h2, { id: "what-a-future-data-project-looks-like", children: [_jsx(_components.a, { className: "heading-link", href: "#what-a-future-data-project-looks-like", children: _jsx(_components.svg, { xmlns: "http:www.w3.org/2000/svg", fill: "#ab2b65", viewBox: "0 0 20 20", className: "w-5 h-5", children: _jsx(_components.path, { fillRule: "evenodd", clipRule: "evenodd", d: "M9.493 2.853a.75.75 0 00-1.486-.205L7.545 6H4.198a.75.75 0 000 1.5h3.14l-.69 5H3.302a.75.75 0 000 1.5h3.14l-.435 3.148a.75.75 0 001.486.205L7.955 14h2.986l-.434 3.148a.75.75 0 001.486.205L12.456 14h3.346a.75.75 0 000-1.5h-3.14l.69-5h3.346a.75.75 0 000-1.5h-3.14l.435-3.147a.75.75 0 00-1.486-.205L12.045 6H9.059l.434-3.147zM8.852 7.5l-.69 5h2.986l.69-5H8.852z" }) }) }), "What a (future) Data Project looks like"] }), "\n", _jsxs(_components.p, { children: ["NB: To understand what a project is see ", _jsx(_components.a, { href: "/dms/dms", children: "DMS" }), "."] }), "\n", _jsx(_components.p, { children: "It helps me to be very concreate and imagine what this looks looks like on disk:" }), "\n", _jsx(_components.pre, { children: _jsxs(_components.code, { className: "language-auto code-highlight", children: [_jsx(_components.span, { className: "code-line", children: "datapackage.json # dataset\n" }), _jsx(_components.span, { className: "code-line", children: "data.csv # dataset resources (could be anywhere)\n" }), _jsx(_components.span, { className: "code-line", children: "views.yml\n" }), _jsx(_components.span, { className: "code-line", children: "data-api.yml\n" }), _jsx(_components.span, { className: "code-line", children: "flows.yml\n" })] }) }), "\n", _jsx(_components.p, { children: "Or, a bit more elegantly:" }), "\n", _jsx(_components.pre, { children: _jsxs(_components.code, { className: "language-auto code-highlight", children: [_jsx(_components.span, { className: "code-line", children: "data/\n" }), _jsx(_components.span, { className: "code-line", children: " gdp.csv | gdp.pdf | ...\n" }), _jsx(_components.span, { className: "code-line", children: "views/\n" }), _jsx(_components.span, { className: "code-line", children: " graph.json | table.json | ...\n" }), _jsx(_components.span, { className: "code-line", children: "api/\n" }), _jsx(_components.span, { className: "code-line", children: " gdp.json | gdp-ppp.json | ...\n" }), _jsx(_components.span, { className: "code-line", children: "flows/\n" }), _jsx(_components.span, { className: "code-line", children: " ...\n" }), _jsx(_components.span, { className: "code-line", children: "README.md\n" }), _jsx(_components.span, { className: "code-line", children: "datapackage.json # ? does this just contain resources or more than that? Just resources\n" })] }) }), "\n", _jsxs(_components.h2, { id: "data-factory-2020-07-23-rufuspollock", children: [_jsx(_components.a, { className: "heading-link", href: "#data-factory-2020-07-23-rufuspollock", children: _jsx(_components.svg, { xmlns: "http:www.w3.org/2000/svg", fill: "#ab2b65", viewBox: "0 0 20 20", className: "w-5 h-5", children: _jsx(_components.path, { fillRule: "evenodd", clipRule: "evenodd", d: "M9.493 2.853a.75.75 0 00-1.486-.205L7.545 6H4.198a.75.75 0 000 1.5h3.14l-.69 5H3.302a.75.75 0 000 1.5h3.14l-.435 3.148a.75.75 0 001.486.205L7.955 14h2.986l-.434 3.148a.75.75 0 001.486.205L12.456 14h3.346a.75.75 0 000-1.5h-3.14l.69-5h3.346a.75.75 0 000-1.5h-3.14l.435-3.147a.75.75 0 00-1.486-.205L12.045 6H9.059l.434-3.147zM8.852 7.5l-.69 5h2.986l.69-5H8.852z" }) }) }), "Data Factory 2020-07-23 @rufuspollock"] }), "\n", _jsx(_components.p, { children: "I've used the term Data Factory but it's not in common use. At the same time, there doesn't seem to be a good term out there in common use for what I'm referring to." }), "\n", _jsx(_components.p, { children: "What am I referring to?" }), "\n", _jsx(_components.p, { children: "I can start with terms we do have decent-ish terminlogy for: data pipelines or data flows." }), "\n", _jsx(_components.p, { children: "Idea is reasonably simple: I'm doing stuff to some data and it involves a series of processing steps which I want to run in order. It may get more complex: rather than a linear sequence of tasks I may have branching and/or parallelization." }), "\n", _jsx(_components.p, { children: "Because data \"flows\" through these steps from one end to the otehr we end up with terminology like \"flow\" or \"pipeline\"." }), "\n", _jsx(_components.p, { children: "Broken down into its components we have two things:" }), "\n", _jsxs(_components.ul, { children: ["\n", _jsx(_components.li, { children: "Tasks: the individual processing nodes, i.e. a single operator on a unit of data (aka Processors / Operators)" }), "\n", _jsx(_components.li, { children: "Pipeline: which combines these tasks into a whole (aka Flow, Graph, DAG …). It is a DAG (directed acycle graph) of processors starting from one or more sources and ending in one or more sinks. Note the simple case of a linear flow is very common." }), "\n"] }), "\n", _jsx(_components.p, { children: "[NB: tasks in an actual flow could either be bespoke to that flow or are configured instances of a template / abstract tasks e.g. load from s3 might be be a template task which as a live task in an actual flow will be configured with a specific bucket.]" }), "\n", _jsx(_components.p, { children: "So far, so good." }), "\n", _jsx(_components.p, { children: "But what is the name for a system (like AirFlow, Luigi) for creating, managing and running data pipelines?" }), "\n", _jsx(_components.p, { children: "My answer: a Data Factory." }), "\n", _jsx(_components.p, { children: "What I don't like with this is that it messes with the metaphor: factories are different from pipelines. If we went with Data Factory then we should talk about \"assembly lines\" rather than \"pipelines\" and \"workers\" (?) rather than tasks. If one stuck with water we'd end up with something like a Data Plant but that sounds weird." }), "\n", _jsx(_components.p, { children: "Analogy: for data we clearly have a file and dataset. And a system for organizing and managing datasets is a data catalog or data management system. So what's the name for the system for processing datasets …" }), "\n", _jsxs(_components.p, { children: ["And finally :checkered_flag: I should mention ", _jsx(_components.a, { href: "https://github.com/datopian/aircan/", target: "_blank", rel: "noopener noreferrer", children: "AirCan" }), ", our own Data Factory system we've been developing built on AirFlow."] }), "\n", _jsxs(_components.h3, { id: "appendix-terminology-match-up", children: [_jsx(_components.a, { className: "heading-link", href: "#appendix-terminology-match-up", children: _jsx(_components.svg, { xmlns: "http:www.w3.org/2000/svg", fill: "#ab2b65", viewBox: "0 0 20 20", className: "w-5 h-5", children: _jsx(_components.path, { fillRule: "evenodd", clipRule: "evenodd", d: "M9.493 2.853a.75.75 0 00-1.486-.205L7.545 6H4.198a.75.75 0 000 1.5h3.14l-.69 5H3.302a.75.75 0 000 1.5h3.14l-.435 3.148a.75.75 0 001.486.205L7.955 14h2.986l-.434 3.148a.75.75 0 001.486.205L12.456 14h3.346a.75.75 0 000-1.5h-3.14l.69-5h3.346a.75.75 0 000-1.5h-3.14l.435-3.147a.75.75 0 00-1.486-.205L12.045 6H9.059l.434-3.147zM8.852 7.5l-.69 5h2.986l.69-5H8.852z" }) }) }), "Appendix: Terminology match up"] }), "\n", _jsx(_components.div, { className: "overflow-x-auto", children: _jsxs(_components.table, { children: [_jsx(_components.thead, { children: _jsxs(_components.tr, { children: [_jsx(_components.th, { children: "Concept" }), _jsx(_components.th, { children: "Airflow" }), _jsx(_components.th, { children: "Luigi" })] }) }), _jsxs(_components.tbody, { children: [_jsxs(_components.tr, { children: [_jsx(_components.td, { children: "Processor" }), _jsx(_components.td, { children: "Task" }), _jsx(_components.td, { children: "Task" })] }), _jsxs(_components.tr, { children: [_jsx(_components.td, { children: "Pipeline" }), _jsx(_components.td, { children: "DAG" }), _jsx(_components.td, { children: "?" })] }), _jsxs(_components.tr, { children: [_jsx(_components.td, { children: "Pipeline (complex, branching)" }), _jsx(_components.td, { children: "DAG" }), _jsx(_components.td, {})] })] })] }) }), "\n", _jsxs(_components.h2, { id: "commonalities-of-harvesting-and-datastore-loading-2020-06-01-rufuspollock", children: [_jsx(_components.a, { className: "heading-link", href: "#commonalities-of-harvesting-and-datastore-loading-2020-06-01-rufuspollock", children: _jsx(_components.svg, { xmlns: "http:www.w3.org/2000/svg", fill: "#ab2b65", viewBox: "0 0 20 20", className: "w-5 h-5", children: _jsx(_components.path, { fillRule: "evenodd", clipRule: "evenodd", d: "M9.493 2.853a.75.75 0 00-1.486-.205L7.545 6H4.198a.75.75 0 000 1.5h3.14l-.69 5H3.302a.75.75 0 000 1.5h3.14l-.435 3.148a.75.75 0 001.486.205L7.955 14h2.986l-.434 3.148a.75.75 0 001.486.205L12.456 14h3.346a.75.75 0 000-1.5h-3.14l.69-5h3.346a.75.75 0 000-1.5h-3.14l.435-3.147a.75.75 0 00-1.486-.205L12.045 6H9.059l.434-3.147zM8.852 7.5l-.69 5h2.986l.69-5H8.852z" }) }) }), "Commonalities of Harvesting and Data(Store) Loading 2020-06-01 @rufuspollock"] }), "\n", _jsx(_components.p, { children: "tags: portal, load, factory" }), "\n", _jsx(_components.p, { children: "Harvesting and data loading (for data API) are almost identical as mechanisms. As such, they can share the same \"data factory\" system." }), "\n", _jsx(_components.p, { children: "Data API load to backing DB (CKAN DataStore + DataPusher stuff)" }), "\n", _jsx(_components.mermaid, { config: "{}", chart: "graph LR\n\nsubgraph Factory\n read --> process\n process --> load\n orchestrator\nend\nload --> db[DB = DataStore]\n\norchestrator --> api\napi --> wui[Dataset Admin UI]" }), "\n", _jsx(_components.p, { children: "Harvesting" }), "\n", _jsx(_components.mermaid, { chart: "graph LR\n\nsubgraph Factory\n read --> process\n process --> load[Load - sync with CKAN]\n orchestrator\nend\nload --> db[DB = MetaStore]\n\norchestrator --> api\napi --> wui[Harvesting Admin UI]" }), "\n", _jsxs(_components.h2, { id: "10-things-i-regret-about-nodejs-by-ryan-dahl-2018-2020-05-17-rufuspollock", children: [_jsx(_components.a, { className: "heading-link", href: "#10-things-i-regret-about-nodejs-by-ryan-dahl-2018-2020-05-17-rufuspollock", children: _jsx(_components.svg, { xmlns: "http:www.w3.org/2000/svg", fill: "#ab2b65", viewBox: "0 0 20 20", className: "w-5 h-5", children: _jsx(_components.path, { fillRule: "evenodd", clipRule: "evenodd", d: "M9.493 2.853a.75.75 0 00-1.486-.205L7.545 6H4.198a.75.75 0 000 1.5h3.14l-.69 5H3.302a.75.75 0 000 1.5h3.14l-.435 3.148a.75.75 0 001.486.205L7.955 14h2.986l-.434 3.148a.75.75 0 001.486.205L12.456 14h3.346a.75.75 0 000-1.5h-3.14l.69-5h3.346a.75.75 0 000-1.5h-3.14l.435-3.147a.75.75 0 00-1.486-.205L12.045 6H9.059l.434-3.147zM8.852 7.5l-.69 5h2.986l.69-5H8.852z" }) }) }), "10 things I regret about NodeJS by Ryan Dahl (2018) 2020-05-17 @rufuspollock"] }), "\n", _jsx(_components.p, { children: "Valuable generally and more great lessons for data packaging." }), "\n", _jsx(_components.div, { style: { position: "relative", paddingBottom: "56.25%" }, children: _jsx(_components.iframe, { style: { position: "absolute", top: "0", left: "0", width: "100%", height: "100%" }, src: "https://www.youtube.com/embed/M3BM9TB-8yA?v=M3BM9TB-8yA", allowFullScreen: true, frameBorder: "0", allow: "accelerometer autoplay clipboard-write encrypted-media gyroscope picture-in-picture" }) }), "\n", _jsxs(_components.h3, { id: "packagejson-and-npm-were-a-mistake", children: [_jsx(_components.a, { className: "heading-link", href: "#packagejson-and-npm-were-a-mistake", children: _jsx(_components.svg, { xmlns: "http:www.w3.org/2000/svg", fill: "#ab2b65", viewBox: "0 0 20 20", className: "w-5 h-5", children: _jsx(_components.path, { fillRule: "evenodd", clipRule: "evenodd", d: "M9.493 2.853a.75.75 0 00-1.486-.205L7.545 6H4.198a.75.75 0 000 1.5h3.14l-.69 5H3.302a.75.75 0 000 1.5h3.14l-.435 3.148a.75.75 0 001.486.205L7.955 14h2.986l-.434 3.148a.75.75 0 001.486.205L12.456 14h3.346a.75.75 0 000-1.5h-3.14l.69-5h3.346a.75.75 0 000-1.5h-3.14l.435-3.147a.75.75 0 00-1.486-.205L12.045 6H9.059l.434-3.147zM8.852 7.5l-.69 5h2.986l.69-5H8.852z" }) }) }), "package.json and npm were a mistake"] }), "\n", _jsxs(_components.p, { children: ["Why package.json was a mistake: ", _jsx(_components.a, { href: "https://youtu.be/M3BM9TB-8yA?t=595", target: "_blank", rel: "noopener noreferrer", children: "https://youtu.be/M3BM9TB-8yA?t=595" })] }), "\n", _jsx(_components.p, { children: _jsx(_components.img, { src: "https://i.imgur.com/Ia0qtVm.png", alt: "" }) }), "\n", _jsxs(_components.ul, { children: ["\n", _jsxs(_components.li, { children: [_jsx(_components.code, { className: "language-auto", children: "npm" }), " + a centralized repo are unnecessary (and were a mistake)"] }), "\n", _jsx(_components.li, { children: "doesn't like centralized npm repo (I agree) and look what go are doing. Sure, you probably have something via the backdoor at the end (e.g. go is getting that) for caching and reliability purposes, but it is not strictly necessary." }), "\n"] }), "\n", _jsxs(_components.p, { children: ["ASIDE: It's the core tool (node) that make a metadata format and packager relevant and successful. It's node require allowing using of ", _jsx(_components.code, { className: "language-auto", children: "package.json" }), " or bundling ", _jsx(_components.code, { className: "language-auto", children: "npm" }), " in by defaultonly"] }), "\n", _jsxs(_components.ul, { children: ["\n", _jsx(_components.li, { children: "Kind of obvious when you think about it" }), "\n", _jsxs(_components.li, { children: ["Something i've always said re Data Packages (but not strongly enough and not followed enough): the tooling comes first and the format is, in many ways, a secondary conveinience. To be fair to myself (and others) we did write ", _jsx(_components.code, { className: "language-auto", children: "dpm" }), " first (in 2007!), and do a lot of stuff with the workflow and toolchain but its easy to get distracted."] }), "\n"] }), "\n", _jsx(_components.p, { children: _jsx(_components.img, { src: "https://i.imgur.com/5E0Hffs.png", alt: "" }) }), "\n", _jsxs(_components.ul, { children: ["\n", _jsxs(_components.li, { children: ["modules aren't strictly necessary and ", _jsx(_components.code, { className: "language-auto", children: "package.json" }), " metadata is not needed"] }), "\n", _jsx(_components.li, { children: "on the web you just have js file and you can include them …" }), "\n", _jsxs(_components.li, { children: ["\"package.json has all this unnecessary noise in it. like license, repository. Why am i filling this out. I feel like a book-keeper or something. This is just unnecessary stuff ", _jsx(_components.em, { children: "to do" }), " when all I am trying to do is link to a library\" [ed: ", _jsx(_components.strong, { children: "I think this is a major relevance for Data Packages. There's a tension between the book-keepers who want lots of metadata for publishing etc … and … the data science and data engineers who just want to get something done. If I had a choice (and I do!) I would prioritize the latter massively. And they just care about stuff like a) table schema b) get me the data on my hard disk fast" }), "]"] }), "\n", _jsxs(_components.li, { children: ["\"If only relative files and URLs were used when importing, the path defines the version. There is no need to list dependencies\" [ed: cf Go that did this right]", "\n", _jsxs(_components.ul, { children: ["\n", _jsx(_components.li, { children: "And he's borrowed from Go for deno.land" }), "\n"] }), "\n"] }), "\n"] }), "\n", _jsxs(_components.h3, { id: "vendoring-by-default-with-node_modules-was-a-mistake", children: [_jsx(_components.a, { className: "heading-link", href: "#vendoring-by-default-with-node_modules-was-a-mistake", children: _jsx(_components.svg, { xmlns: "http:www.w3.org/2000/svg", fill: "#ab2b65", viewBox: "0 0 20 20", className: "w-5 h-5", children: _jsx(_components.path, { fillRule: "evenodd", clipRule: "evenodd", d: "M9.493 2.853a.75.75 0 00-1.486-.205L7.545 6H4.198a.75.75 0 000 1.5h3.14l-.69 5H3.302a.75.75 0 000 1.5h3.14l-.435 3.148a.75.75 0 001.486.205L7.955 14h2.986l-.434 3.148a.75.75 0 001.486.205L12.456 14h3.346a.75.75 0 000-1.5h-3.14l.69-5h3.346a.75.75 0 000-1.5h-3.14l.435-3.147a.75.75 0 00-1.486-.205L12.045 6H9.059l.434-3.147zM8.852 7.5l-.69 5h2.986l.69-5H8.852z" }) }) }), "Vendoring by default with ", _jsx(_components.code, { className: "language-auto", children: "node_modules" }), " was a mistake"] }), "\n", _jsxs(_components.p, { children: ["Vendoring by default with ", _jsx(_components.code, { className: "language-auto", children: "node_modules" }), " was a mistake - just use an env variable ", _jsx(_components.code, { className: "language-auto", children: "$NODE_PATH" })] }), "\n", _jsxs(_components.ul, { children: ["\n", _jsxs(_components.li, { children: [_jsx(_components.code, { className: "language-auto", children: "node_modules" }), " then becomes massive"] }), "\n", _jsx(_components.li, { children: "module resolution is (needlessly) complex" }), "\n"] }), "\n", _jsxs(_components.h3, { id: "general-point-kiss--yagni", children: [_jsx(_components.a, { className: "heading-link", href: "#general-point-kiss--yagni", children: _jsx(_components.svg, { xmlns: "http:www.w3.org/2000/svg", fill: "#ab2b65", viewBox: "0 0 20 20", className: "w-5 h-5", children: _jsx(_components.path, { fillRule: "evenodd", clipRule: "evenodd", d: "M9.493 2.853a.75.75 0 00-1.486-.205L7.545 6H4.198a.75.75 0 000 1.5h3.14l-.69 5H3.302a.75.75 0 000 1.5h3.14l-.435 3.148a.75.75 0 001.486.205L7.955 14h2.986l-.434 3.148a.75.75 0 001.486.205L12.456 14h3.346a.75.75 0 000-1.5h-3.14l.69-5h3.346a.75.75 0 000-1.5h-3.14l.435-3.147a.75.75 0 00-1.486-.205L12.045 6H9.059l.434-3.147zM8.852 7.5l-.69 5h2.986l.69-5H8.852z" }) }) }), "General point: KISS / YAGNI"] }), "\n", _jsxs(_components.p, { children: ["E.g. ", _jsx(_components.code, { className: "language-auto", children: "index.js" }), "was \"cute\" but unnecessary, allowing ", _jsx(_components.code, { className: "language-auto", children: "require xxx" }), " without an extension (e.g. ", _jsx(_components.code, { className: "language-auto", children: ".js" }), " or ", _jsx(_components.code, { className: "language-auto", children: ".ts" }), " ) means you have to probe the file system."] }), "\n", _jsx(_components.p, { children: "+data package. +data packaging. +frictionless. +lessons" }), "\n", _jsxs(_components.h2, { id: "go-modules-and-dependency-management-re-data-package-management-2020-05-16-rufuspollock", children: [_jsx(_components.a, { className: "heading-link", href: "#go-modules-and-dependency-management-re-data-package-management-2020-05-16-rufuspollock", children: _jsx(_components.svg, { xmlns: "http:www.w3.org/2000/svg", fill: "#ab2b65", viewBox: "0 0 20 20", className: "w-5 h-5", children: _jsx(_components.path, { fillRule: "evenodd", clipRule: "evenodd", d: "M9.493 2.853a.75.75 0 00-1.486-.205L7.545 6H4.198a.75.75 0 000 1.5h3.14l-.69 5H3.302a.75.75 0 000 1.5h3.14l-.435 3.148a.75.75 0 001.486.205L7.955 14h2.986l-.434 3.148a.75.75 0 001.486.205L12.456 14h3.346a.75.75 0 000-1.5h-3.14l.69-5h3.346a.75.75 0 000-1.5h-3.14l.435-3.147a.75.75 0 00-1.486-.205L12.045 6H9.059l.434-3.147zM8.852 7.5l-.69 5h2.986l.69-5H8.852z" }) }) }), "Go modules and dependency management (re data package management) 2020-05-16 @rufuspollock"] }), "\n", _jsx(_components.p, { children: "Generally Go does stuff well. They also punted on dependency management initially. First, you just installed a url it was up to you to manage your depedencies. Then there was a period of chaos as several package/dependency managers fought it out (GoDeps etc). Then, ~ 2018 the community came together led by Russ Cox and came up with a very solid proposal which is official as of 2019." }), "\n", _jsx(_components.p, { children: "Go's approach to module (package) and dependency management can be an inspiration for Frictionless and Data Packages. Just as we learnt and borrowed a lot from Python and Node so we can learn and borrow from Go." }), "\n", _jsxs(_components.ol, { children: ["\n", _jsxs(_components.li, { children: ["Overview (by Russ Cox the author): ", _jsx(_components.a, { href: "https://research.swtch.com/vgo", target: "_blank", rel: "noopener noreferrer", children: "https://research.swtch.com/vgo" })] }), "\n", _jsxs(_components.li, { children: ["The Principles of Versioning in Go ", _jsx(_components.a, { href: "https://research.swtch.com/vgo-principles", target: "_blank", rel: "noopener noreferrer", children: "https://research.swtch.com/vgo-principles" })] }), "\n", _jsxs(_components.li, { children: ["A Tour of Versioned Go (vgo) ", _jsx(_components.a, { href: "https://research.swtch.com/vgo-tour", target: "_blank", rel: "noopener noreferrer", children: "https://research.swtch.com/vgo-tour" })] }), "\n", _jsxs(_components.li, { children: ["cmd/go: add package version support to Go toolchain ", _jsx(_components.a, { href: "https://github.com/golang/go/issues/24301", target: "_blank", rel: "noopener noreferrer", children: "https://github.com/golang/go/issues/24301" })] }), "\n", _jsxs(_components.li, { children: ["Using Go Modules - ", _jsx(_components.a, { href: "https://blog.golang.org/using-go-modules", target: "_blank", rel: "noopener noreferrer", children: "https://blog.golang.org/using-go-modules" }), " (official introduction on go blog)"] }), "\n", _jsxs(_components.li, { children: ["Publishing Go Modules ", _jsx(_components.a, { href: "https://blog.golang.org/publishing-go-modules", target: "_blank", rel: "noopener noreferrer", children: "https://blog.golang.org/publishing-go-modules" })] }), "\n", _jsxs(_components.li, { children: ["Main wiki article and overview ", _jsx(_components.a, { href: "https://github.com/golang/go/wiki/Modules", target: "_blank", rel: "noopener noreferrer", children: "https://github.com/golang/go/wiki/Modules" })] }), "\n"] }), "\n", _jsxs(_components.h3, { id: "key-principles", children: [_jsx(_components.a, { className: "heading-link", href: "#key-principles", children: _jsx(_components.svg, { xmlns: "http:www.w3.org/2000/svg", fill: "#ab2b65", viewBox: "0 0 20 20", className: "w-5 h-5", children: _jsx(_components.path, { fillRule: "evenodd", clipRule: "evenodd", d: "M9.493 2.853a.75.75 0 00-1.486-.205L7.545 6H4.198a.75.75 0 000 1.5h3.14l-.69 5H3.302a.75.75 0 000 1.5h3.14l-.435 3.148a.75.75 0 001.486.205L7.955 14h2.986l-.434 3.148a.75.75 0 001.486.205L12.456 14h3.346a.75.75 0 000-1.5h-3.14l.69-5h3.346a.75.75 0 000-1.5h-3.14l.435-3.147a.75.75 0 00-1.486-.205L12.045 6H9.059l.434-3.147zM8.852 7.5l-.69 5h2.986l.69-5H8.852z" }) }) }), "Key principles"] }), "\n", _jsxs(_components.blockquote, { children: ["\n", _jsx(_components.p, { children: "These are the three principles of versioning in Go, the reasons that the design of Go modules deviates from the design of Dep, Cargo, Bundler, and others." }), "\n", _jsxs(_components.ol, { children: ["\n", _jsx(_components.li, { children: "Compatibility. The meaning of a name in a program should not change over time." }), "\n", _jsxs(_components.li, { children: ["Repeatability. The result of a build of a given version of a package should not change over time. ", _jsx(_components.a, { href: "https://research.swtch.com/vgo-principles#repeatability", target: "_blank", rel: "noopener noreferrer", children: "https://research.swtch.com/vgo-principles#repeatability" })] }), "\n", _jsx(_components.li, { children: "Cooperation. To maintain the Go package ecosystem, we must all work together. Tools cannot work around a lack of cooperation." }), "\n"] }), "\n"] }), "\n", _jsx(_components.p, { children: "Summary" }), "\n", _jsxs(_components.ul, { children: ["\n", _jsxs(_components.li, { children: ["Go used urls for identifiers for packages (including special cases for github)", "\n", _jsxs(_components.ul, { children: ["\n", _jsxs(_components.li, { children: ["e.g. ", _jsx(_components.code, { className: "language-auto", children: "import rsc.io/quote" })] }), "\n", _jsx(_components.li, { children: "Brilliant! No more dependency resolution via some central service. Just use the internet." }), "\n"] }), "\n"] }), "\n", _jsxs(_components.li, { children: ["Go installed packages via ", _jsx(_components.code, { className: "language-auto", children: "go get" }), " e.g. ", _jsx(_components.code, { className: "language-auto", children: "go get rsc.io/quote" }), ". This would install the module into ", _jsx(_components.code, { className: "language-auto", children: "$GOPATH" }), " at ", _jsx(_components.code, { className: "language-auto", children: "rsc.io/quote" }), "\n", _jsxs(_components.ul, { children: ["\n", _jsxs(_components.li, { children: ["They did the absolute minimum: grab the files onto your hard disk under ", _jsx(_components.code, { className: "language-auto", children: "$GOPATH/src" }), ". ", _jsx(_components.code, { className: "language-auto", children: "import" }), " would then search this (IIUC)"] }), "\n"] }), "\n"] }), "\n", _jsxs(_components.li, { children: ["There was no way originally to get a version but with go modules (go > 1.11) you could do ", _jsx(_components.code, { className: "language-auto", children: "go get rsc.io/quote@[version]" })] }), "\n", _jsxs(_components.li, { children: ["Dependency management is actually complex: satisfying dependency requirements is NP complete. Solve this by …", "\n", _jsxs(_components.ul, { children: ["\n", _jsxs(_components.li, { children: ["\n", _jsx(_components.p, { children: "The Node/Bundler/Cargo/Dep approach is one answer. Allow authors to specify arbitrary constraints on their dependencies. Build a given target by collecting all its dependencies recursively and finding a configuration satisfying all those constraints. => SAT solver => this is complex." }), "\n"] }), "\n", _jsxs(_components.li, { children: ["\n", _jsx(_components.p, { children: "Go has a different solution" }), "\n"] }), "\n", _jsxs(_components.li, { children: ["\n", _jsx(_components.p, { children: "Versioning is central to dependency management => you need to get really clear on versioning. Establish a community rule that you can only break compatibility with major versions …" }), "\n"] }), "\n", _jsxs(_components.li, { children: ["\n", _jsxs(_components.p, { children: ["Put breaking version (e.g. major versions) ", _jsx(_components.strong, { children: "into the url" }), " so that you actually have a different package …"] }), "\n", _jsxs(_components.blockquote, { children: ["\n", _jsx(_components.p, { children: "For Go modules, we gave this old advice a new name, the import compatibility rule:" }), "\n"] }), "\n", _jsxs(_components.blockquote, { children: ["\n", _jsxs(_components.blockquote, { children: ["\n", _jsx(_components.p, { children: "If an old package and a new package have the same import path,\nthe new package must be backwards compatible with the old package." }), "\n"] }), "\n"] }), "\n", _jsx(_components.p, { children: _jsx(_components.img, { src: "https://research.swtch.com/impver@2x.png", alt: "" }) }), "\n"] }), "\n", _jsxs(_components.li, { children: ["\n", _jsx(_components.p, { children: "Install the minimal version of a package that satisfies the requirements (rather than the maximal version) => this yields repeatability (principle 2)" }), "\n"] }), "\n", _jsxs(_components.li, { children: ["\n", _jsxs(_components.p, { children: ["In summary Go differs in that: all versions are explicit (no ", _jsx(_components.code, { className: "language-auto", children: "<=" }), ", ", _jsx(_components.code, { className: "language-auto", children: ">=" }), "). Since we can assume that all later versions of a module are backwards compatible (and any breaking change generates a new module with explicit ", _jsx(_components.code, { className: "language-auto", children: "vX" }), " in name) we can simply cycle through a module and its dependencies and find the highest version that are listed and install that."] }), "\n"] }), "\n"] }), "\n"] }), "\n", _jsxs(_components.li, { children: ["Publishing a module is just pushing to github/gitlab or putting it somewhere on the web – see ", _jsx(_components.a, { href: "https://blog.golang.org/publishing-go-modules", target: "_blank", rel: "noopener noreferrer", children: "https://blog.golang.org/publishing-go-modules" })] }), "\n", _jsx(_components.li, { children: "Tagging versions is done with git tag" }), "\n", _jsx(_components.li, { children: "\"A module is a collection of related Go packages that are versioned together as a single unit.\"" }), "\n"] }), "\n", _jsxs(_components.p, { children: ["Layout on disk in a module (see e.g. ", _jsx(_components.a, { href: "https://blog.golang.org/publishing-go-modules", target: "_blank", rel: "noopener noreferrer", children: "https://blog.golang.org/publishing-go-modules" }), "). Main file ", _jsx(_components.code, { className: "language-auto", children: "go.mod" }), " and one extra for storing hashes for verification (it's not a lock file)"] }), "\n", _jsx(_components.pre, { children: _jsxs(_components.code, { className: "language-auto code-highlight", children: [_jsx(_components.span, { className: "code-line", children: "$ cat go.mod\n" }), _jsx(_components.span, { className: "code-line", children: "module example.com/hello\n" }), _jsx(_components.span, { className: "code-line", children: "\n" }), _jsx(_components.span, { className: "code-line", children: "go 1.12\n" }), _jsx(_components.span, { className: "code-line", children: "\n" }), _jsx(_components.span, { className: "code-line", children: "require rsc.io/quote/v3 v3.1.0\n" }), _jsx(_components.span, { className: "code-line", children: "\n" }), _jsx(_components.span, { className: "code-line", children: "$ cat go.sum\n" }), _jsx(_components.span, { className: "code-line", children: "golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c h1:qgOY6WgZOaTkIIMiVjBQcw93ERBE4m30iBm00nkL0i8=\n" }), _jsx(_components.span, { className: "code-line", children: "golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=\n" }), _jsx(_components.span, { className: "code-line", children: "rsc.io/quote/v3 v3.1.0 h1:9JKUTTIUgS6kzR9mK1YuGKv6Nl+DijDNIc0ghT58FaY=\n" }), _jsx(_components.span, { className: "code-line", children: "rsc.io/quote/v3 v3.1.0/go.mod h1:yEA65RcK8LyAZtP9Kv3t0HmxON59tX3rD+tICJqUlj0=\n" }), _jsx(_components.span, { className: "code-line", children: "rsc.io/sampler v1.3.0 h1:7uVkIFmeBqHfdjD+gZwtXXI+RODJ2Wc4O7MPEh/QiW4=\n" }), _jsx(_components.span, { className: "code-line", children: "rsc.io/sampler v1.3.0/go.mod h1:T1hPZKmBbMNahiBKFy5HrXp6adAjACjK9JXDnKaTXpA=\n" })] }) }), "\n", _jsxs(_components.h3, { id: "asides", children: [_jsx(_components.a, { className: "heading-link", href: "#asides", children: _jsx(_components.svg, { xmlns: "http:www.w3.org/2000/svg", fill: "#ab2b65", viewBox: "0 0 20 20", className: "w-5 h-5", children: _jsx(_components.path, { fillRule: "evenodd", clipRule: "evenodd", d: "M9.493 2.853a.75.75 0 00-1.486-.205L7.545 6H4.198a.75.75 0 000 1.5h3.14l-.69 5H3.302a.75.75 0 000 1.5h3.14l-.435 3.148a.75.75 0 001.486.205L7.955 14h2.986l-.434 3.148a.75.75 0 001.486.205L12.456 14h3.346a.75.75 0 000-1.5h-3.14l.69-5h3.346a.75.75 0 000-1.5h-3.14l.435-3.147a.75.75 0 00-1.486-.205L12.045 6H9.059l.434-3.147zM8.852 7.5l-.69 5h2.986l.69-5H8.852z" }) }) }), "Asides"] }), "\n", _jsx(_components.p, { children: "#### Vendoring is an incomplete solution to package versioning problem" }), "\n", _jsxs(_components.blockquote, { children: ["\n", _jsxs(_components.p, { children: ["More fundamentally, vendoring is an incomplete solution to the package versioning problem. It only provides reproducible builds. It does nothing to help understand package versions and decide which version of a package to use. Package managers like glide and dep add the concept of versioning onto Go builds implicitly, without direct toolchain support, by setting up the vendor directory a certain way. As a result, the many tools in the Go ecosystem cannot be made properly aware of versions. It's clear that Go needs direct toolchain support for package versions. ", _jsx(_components.a, { href: "https://research.swtch.com/vgo-intro", target: "_blank", rel: "noopener noreferrer", children: "https://research.swtch.com/vgo-intro" })] }), "\n"] }), "\n", _jsxs(_components.h2, { id: "2020-05-16-rufuspollock", children: [_jsx(_components.a, { className: "heading-link", href: "#2020-05-16-rufuspollock", children: _jsx(_components.svg, { xmlns: "http:www.w3.org/2000/svg", fill: "#ab2b65", viewBox: "0 0 20 20", className: "w-5 h-5", children: _jsx(_components.path, { fillRule: "evenodd", clipRule: "evenodd", d: "M9.493 2.853a.75.75 0 00-1.486-.205L7.545 6H4.198a.75.75 0 000 1.5h3.14l-.69 5H3.302a.75.75 0 000 1.5h3.14l-.435 3.148a.75.75 0 001.486.205L7.955 14h2.986l-.434 3.148a.75.75 0 001.486.205L12.456 14h3.346a.75.75 0 000-1.5h-3.14l.69-5h3.346a.75.75 0 000-1.5h-3.14l.435-3.147a.75.75 0 00-1.486-.205L12.045 6H9.059l.434-3.147zM8.852 7.5l-.69 5h2.986l.69-5H8.852z" }) }) }), "2020-05-16 @rufuspollock"] }), "\n", _jsx(_components.p, { children: "Ruthlessly retain compatibility after v1 - inspiration from Go for Frictionless" }), "\n", _jsxs(_components.blockquote, { children: ["\n", _jsx(_components.p, { children: "It is intended that programs written to the Go 1 specification will continue to compile and run correctly, unchanged, over the lifetime of that specification. Go programs that work today should continue to work even as future “point” releases of Go 1 arise (Go 1.1, Go 1.2, etc.)." }), "\n", _jsxs(_components.p, { children: ["— ", _jsx(_components.a, { href: "https://golang.org/doc/go1compat", target: "_blank", rel: "noopener noreferrer", children: "https://golang.org/doc/go1compat" })] }), "\n"] }), "\n", _jsx(_components.p, { children: "And they go further – not just Go but also Go packages:" }), "\n", _jsxs(_components.blockquote, { children: ["\n", _jsx(_components.p, { children: "Packages intended for public use should try to maintain backwards compatibility as they evolve. The Go 1 compatibility guidelines are a good reference here: don’t remove exported names, encourage tagged composite literals, and so on. If different functionality is required, add a new name instead of changing an old one. If a complete break is required, create a new package with a new import path." }), "\n", _jsx(_components.p, { children: "The Go FAQ has since Go 1.2 in November 2013" }), "\n"] }), "\n", _jsx(_components.p, { children: "+frictionless" }), "\n", _jsxs(_components.h2, { id: "2020-05-15-rufuspollock", children: [_jsx(_components.a, { className: "heading-link", href: "#2020-05-15-rufuspollock", children: _jsx(_components.svg, { xmlns: "http:www.w3.org/2000/svg", fill: "#ab2b65", viewBox: "0 0 20 20", className: "w-5 h-5", children: _jsx(_components.path, { fillRule: "evenodd", clipRule: "evenodd", d: "M9.493 2.853a.75.75 0 00-1.486-.205L7.545 6H4.198a.75.75 0 000 1.5h3.14l-.69 5H3.302a.75.75 0 000 1.5h3.14l-.435 3.148a.75.75 0 001.486.205L7.955 14h2.986l-.434 3.148a.75.75 0 001.486.205L12.456 14h3.346a.75.75 0 000-1.5h-3.14l.69-5h3.346a.75.75 0 000-1.5h-3.14l.435-3.147a.75.75 0 00-1.486-.205L12.045 6H9.059l.434-3.147zM8.852 7.5l-.69 5h2.986l.69-5H8.852z" }) }) }), "2020-05-15 @rufuspollock"] }), "\n", _jsxs(_components.p, { children: [_jsx(_components.code, { className: "language-auto", children: "Project" }), " should be the primary object in a DataHub/Data Portal – not Dataset."] }), "\n", _jsx(_components.p, { children: "Why? Because actually this is more than a Dataset. For example, it includes issues or workflows. A project is a good name for this that is both generic and specific." }), "\n", _jsx(_components.p, { children: "cf Git-hub e.g. Gitlab (and Github). Gitlab came later and did this right: it's primary object is a Project which hasA Repository. Github still insits on calling them repositories (see primary menu item which is \"Create a new repository\"). This is weird, a github \"repository\" isn't actually a github repository: it has issues, stats, workflows and even a discussion board now. Calling it a project is the accurate description and the repository label is a historical artifact when that was all it was. I sometimes create \"repos\" on Github just to have an issue tracker. Gitlab understands this and actually allows me to have projects without any associated repository." }), "\n", _jsx(_components.p, { children: "TODO: take a screenshot to illustrate Gitlab and Github." }), "\n", _jsx(_components.p, { children: "+flashes of insight. +datahub +data portal. +domain model." }), "\n", _jsxs(_components.h2, { id: "2020-04-23-rufuspollock", children: [_jsx(_components.a, { className: "heading-link", href: "#2020-04-23-rufuspollock", children: _jsx(_components.svg, { xmlns: "http:www.w3.org/2000/svg", fill: "#ab2b65", viewBox: "0 0 20 20", className: "w-5 h-5", children: _jsx(_components.path, { fillRule: "evenodd", clipRule: "evenodd", d: "M9.493 2.853a.75.75 0 00-1.486-.205L7.545 6H4.198a.75.75 0 000 1.5h3.14l-.69 5H3.302a.75.75 0 000 1.5h3.14l-.435 3.148a.75.75 0 001.486.205L7.955 14h2.986l-.434 3.148a.75.75 0 001.486.205L12.456 14h3.346a.75.75 0 000-1.5h-3.14l.69-5h3.346a.75.75 0 000-1.5h-3.14l.435-3.147a.75.75 0 00-1.486-.205L12.045 6H9.059l.434-3.147zM8.852 7.5l-.69 5h2.986l.69-5H8.852z" }) }) }), "2020-04-23 @rufuspollock"] }), "\n", _jsx(_components.p, { children: "4 Stores of a DataHub" }), "\n", _jsxs(_components.blockquote, { children: ["\n", _jsx(_components.p, { children: "[!tip]Naming is one of the most important things – and hardest!" }), "\n"] }), "\n", _jsxs(_components.ul, { children: ["\n", _jsx(_components.li, { children: "MetaStore [service]: API (and store) of the metadata for datasets" }), "\n", _jsx(_components.li, { children: "HubStore [service]: API for registry of datasets (+ potentially organizations and ownership relationships to datasets)" }), "\n", _jsx(_components.li, { children: "BlobStore [service]: API for blobs of data" }), "\n", _jsx(_components.li, { children: "StructuredStore [service]: API for structured data" }), "\n"] }), "\n", _jsx(_components.p, { children: "Origins:" }), "\n", _jsxs(_components.ul, { children: ["\n", _jsx(_components.li, { children: "Started using MetaStore in DataHub.io back in 2016" }), "\n", _jsx(_components.li, { children: "Not used in CKAN v2" }), "\n", _jsx(_components.li, { children: "Conceptually CKAN originally was MetaStore and HubStore." }), "\n"] }), "\n", _jsx(_components.p, { children: "In CKAN v2:" }), "\n", _jsxs(_components.ul, { children: ["\n", _jsx(_components.li, { children: "MetaStore and HubStore (no explicit name) => main Postgres DB" }), "\n", _jsx(_components.li, { children: "StructuredStore (called DataStore) => another separate Postgres DB" }), "\n", _jsx(_components.li, { children: "BlobStore (called FileStore) => local disk (or cloud with an extension)" }), "\n"] }), "\n", _jsx(_components.p, { children: "In CKAN v3: propose to separate these explicitly …" }), "\n", _jsxs(_components.h2, { id: "data-portal-vs-datahub-vs-data-os-2020-04-23-rufuspollock", children: [_jsx(_components.a, { className: "heading-link", href: "#data-portal-vs-datahub-vs-data-os-2020-04-23-rufuspollock", children: _jsx(_components.svg, { xmlns: "http:www.w3.org/2000/svg", fill: "#ab2b65", viewBox: "0 0 20 20", className: "w-5 h-5", children: _jsx(_components.path, { fillRule: "evenodd", clipRule: "evenodd", d: "M9.493 2.853a.75.75 0 00-1.486-.205L7.545 6H4.198a.75.75 0 000 1.5h3.14l-.69 5H3.302a.75.75 0 000 1.5h3.14l-.435 3.148a.75.75 0 001.486.205L7.955 14h2.986l-.434 3.148a.75.75 0 001.486.205L12.456 14h3.346a.75.75 0 000-1.5h-3.14l.69-5h3.346a.75.75 0 000-1.5h-3.14l.435-3.147a.75.75 0 00-1.486-.205L12.045 6H9.059l.434-3.147zM8.852 7.5l-.69 5h2.986l.69-5H8.852z" }) }) }), "Data Portal vs DataHub vs Data OS 2020-04-23 @rufuspollock"] }), "\n", _jsx(_components.p, { children: "Data Portal vs DataHub vs Data OS – naming and definitions." }), "\n", _jsx(_components.p, { children: "Is a Data Portal a DataHub? Is a DataHub a DataOS? If not, what are the differences?" }), "\n", _jsx(_components.p, { children: "+todo" }), "\n", _jsxs(_components.h2, { id: "data-concepts---from-atoms-to-organisms-2020-03-05-rufuspollock", children: [_jsx(_components.a, { className: "heading-link", href: "#data-concepts---from-atoms-to-organisms-2020-03-05-rufuspollock", children: _jsx(_components.svg, { xmlns: "http:www.w3.org/2000/svg", fill: "#ab2b65", viewBox: "0 0 20 20", className: "w-5 h-5", children: _jsx(_components.path, { fillRule: "evenodd", clipRule: "evenodd", d: "M9.493 2.853a.75.75 0 00-1.486-.205L7.545 6H4.198a.75.75 0 000 1.5h3.14l-.69 5H3.302a.75.75 0 000 1.5h3.14l-.435 3.148a.75.75 0 001.486.205L7.955 14h2.986l-.434 3.148a.75.75 0 001.486.205L12.456 14h3.346a.75.75 0 000-1.5h-3.14l.69-5h3.346a.75.75 0 000-1.5h-3.14l.435-3.147a.75.75 0 00-1.486-.205L12.045 6H9.059l.434-3.147zM8.852 7.5l-.69 5h2.986l.69-5H8.852z" }) }) }), "Data Concepts - from Atoms to Organisms 2020-03-05 @rufuspollock"] }), "\n", _jsx(_components.pre, { children: _jsxs(_components.code, { className: "language-auto code-highlight", children: [_jsx(_components.span, { className: "code-line", children: "Point -> Line -> Plane (0d -> 1d -> 2d -> 3d)\n" }), _jsx(_components.span, { className: "code-line", children: "\n" }), _jsx(_components.span, { className: "code-line", children: "Atom -> Molecule -> Cell -> Organism\n" })] }) }), "\n", _jsx(_components.mermaid, { chart: "graph TD\n\ncell[Cell]\nrow[Row]\ncolumn[Column]\ntable[\"Table (Sheet)\"]\ncube[\"Database (Dataset)\"]\n\ncell --> row\ncell --> column\nrow --> table\ncolumn --> table\ntable --> cube" }), "\n", _jsx(_components.pre, { children: _jsxs(_components.code, { className: "language-auto code-highlight", children: [_jsx(_components.span, { className: "code-line", children: " Domain =>\n" }), _jsx(_components.span, { className: "code-line", children: "Dimension\n" }), _jsx(_components.span, { className: "code-line", children: " |\n" }), _jsx(_components.span, { className: "code-line", children: " V\n" })] }) }), "\n", _jsx(_components.div, { className: "overflow-x-auto", children: _jsxs(_components.table, { children: [_jsx(_components.thead, { children: _jsxs(_components.tr, { children: [_jsx(_components.th, { children: "Dimension" }), _jsx(_components.th, { children: "…" }), _jsx(_components.th, { children: "Math" }), _jsx(_components.th, { children: "Spreadsheets" }), _jsx(_components.th, { children: "Databases" }), _jsx(_components.th, { children: "Tables etc" }), _jsx(_components.th, { children: "Frictionless" }), _jsx(_components.th, { children: "Pandas" }), _jsx(_components.th, { children: "R" })] }) }), _jsxs(_components.tbody, { children: [_jsxs(_components.tr, { children: [_jsx(_components.td, { children: "0d" }), _jsx(_components.td, { children: "Datum" }), _jsx(_components.td, { children: "Value" }), _jsx(_components.td, { children: "Cell" }), _jsx(_components.td, { children: "Value" }), _jsx(_components.td, { children: "Scalar?" }), _jsx(_components.td, { children: "N/A" }), _jsx(_components.td, { children: "Value" }), _jsx(_components.td, { children: "?" })] }), _jsxs(_components.tr, { children: [_jsx(_components.td, { children: "1d" }), _jsx(_components.td, { children: "…" }), _jsx(_components.td, { children: "Array / Vector" }), _jsx(_components.td, { children: "Row" }), _jsx(_components.td, { children: "Row" }), _jsx(_components.td, { children: "Row" }), _jsx(_components.td, { children: "N/A" }), _jsx(_components.td, {}), _jsx(_components.td, {})] }), _jsxs(_components.tr, { children: [_jsx(_components.td, { children: "1d" }), _jsx(_components.td, { children: "…" }), _jsx(_components.td, { children: "N/A" }), _jsx(_components.td, { children: "Column" }), _jsx(_components.td, {}), _jsx(_components.td, {}), _jsx(_components.td, {}), _jsx(_components.td, {}), _jsx(_components.td, {})] }), _jsxs(_components.tr, { children: [_jsx(_components.td, { children: "2d" }), _jsx(_components.td, { children: "Grid?" }), _jsx(_components.td, { children: "Matrix" }), _jsx(_components.td, { children: "Sheet" }), _jsx(_components.td, { children: "Table" }), _jsx(_components.td, { children: "Table" }), _jsx(_components.td, { children: "TableSchema" }), _jsx(_components.td, {}), _jsx(_components.td, {})] }), _jsxs(_components.tr, { children: [_jsx(_components.td, { children: "3d" }), _jsx(_components.td, { children: "Cube" }), _jsx(_components.td, { children: "3d Matrix" }), _jsx(_components.td, { children: "Spreadsheet" }), _jsx(_components.td, { children: "Database (or Cube)" }), _jsx(_components.td, { children: "N/A" }), _jsx(_components.td, { children: "?" }), _jsx(_components.td, { children: "?" }), _jsx(_components.td, { children: "?" })] }), _jsxs(_components.tr, { children: [_jsx(_components.td, { children: "4d+" }), _jsx(_components.td, { children: "HyperCube" }), _jsx(_components.td, { children: "n-d Matrix" }), _jsx(_components.td, {}), _jsx(_components.td, {}), _jsx(_components.td, {}), _jsx(_components.td, {}), _jsx(_components.td, {}), _jsx(_components.td, {})] })] })] }) }), "\n", _jsx(_components.p, { children: "What's crucial about a table is that it is not just an array of arrays or a rowset but a rowset plus a fieldset." }), "\n", _jsx(_components.pre, { children: _jsxs(_components.code, { className: "language-auto code-highlight", children: [_jsx(_components.span, { className: "code-line", children: "Field => FieldSet\n" }), _jsx(_components.span, { className: "code-line", children: "Row => RowSet\n" })] }) }), "\n", _jsx(_components.p, { children: "A Table is a FieldSet x RowSet (+ other information)" }), "\n", _jsx(_components.p, { children: "There is the question of whether there is some kind of connection or commonality at each dimension up … e.g. you could have an array of arrays where each array has different fields …" }), "\n", _jsx(_components.pre, { className: "language-json", children: _jsxs(_components.code, { className: "language-json code-highlight", children: [_jsxs(_components.span, { className: "code-line", children: [_jsx(_components.span, { className: "token punctuation", children: "{" }), " ", _jsx(_components.span, { className: "token property", children: "\"first\"" }), _jsx(_components.span, { className: "token operator", children: ":" }), " ", _jsx(_components.span, { className: "token string", children: "\"joe\"" }), _jsx(_components.span, { className: "token punctuation", children: "," }), " ", _jsx(_components.span, { className: "token property", children: "\"height\"" }), _jsx(_components.span, { className: "token operator", children: ":" }), " ", _jsx(_components.span, { className: "token number", children: "3" }), " ", _jsx(_components.span, { className: "token punctuation", children: "}" }), "\n"] }), _jsxs(_components.span, { className: "code-line", children: [_jsx(_components.span, { className: "token punctuation", children: "{" }), " ", _jsx(_components.span, { className: "token property", children: "\"last\"" }), _jsx(_components.span, { className: "token operator", children: ":" }), " ", _jsx(_components.span, { className: "token string", children: "\"gautama\"" }), _jsx(_components.span, { className: "token punctuation", children: "," }), " ", _jsx(_components.span, { className: "token property", children: "\"weight\"" }), _jsx(_components.span, { className: "token operator", children: ":" }), " ", _jsx(_components.span, { className: "token number", children: "50" }), " ", _jsx(_components.span, { className: "token punctuation", children: "}" }), "\n"] })] }) }), "\n", _jsx(_components.p, { children: "But a table has common fields." }), "\n", _jsx(_components.pre, { className: "language-json", children: _jsxs(_components.code, { className: "language-json code-highlight", children: [_jsxs(_components.span, { className: "code-line", children: [_jsx(_components.span, { className: "token punctuation", children: "{" }), " ", _jsx(_components.span, { className: "token property", children: "\"first\"" }), _jsx(_components.span, { className: "token operator", children: ":" }), " ", _jsx(_components.span, { className: "token string", children: "\"joe\"" }), _jsx(_components.span, { className: "token punctuation", children: "," }), " ", _jsx(_components.span, { className: "token property", children: "\"height\"" }), _jsx(_components.span, { className: "token operator", children: ":" }), " ", _jsx(_components.span, { className: "token number", children: "3" }), " ", _jsx(_components.span, { className: "token punctuation", children: "}" }), "\n"] }), _jsxs(_components.span, { className: "code-line", children: [_jsx(_components.span, { className: "token punctuation", children: "{" }), " ", _jsx(_components.span, { className: "token property", children: "\"first\"" }), _jsx(_components.span, { className: "token operator", children: ":" }), " ", _jsx(_components.span, { className: "token string", children: "\"siddarta\"" }), _jsx(_components.span, { className: "token punctuation", children: "," }), " ", _jsx(_components.span, { className: "token property", children: "\"height\"" }), _jsx(_components.span, { className: "token operator", children: ":" }), " ", _jsx(_components.span, { className: "token number", children: "50" }), " ", _jsx(_components.span, { className: "token punctuation", children: "}" }), "\n"] })] }) }), "\n", _jsx(_components.p, { children: "(NB: one could always force a group of rows with disparate fields into being a table by creating the union of all the fields but that's hacky)" }), "\n", _jsx(_components.p, { children: "So a table is a RowSet plus a FieldSet where each row conforms to that FieldSet." }), "\n", _jsxs(_components.p, { children: ["Similarly, we can aggregate tables. By default here the tables do ", _jsx(_components.em, { children: "not" }), " share any commonality – sheets in a spreadsheet need not share any common aspect, nor do tables in a database. If they do, then we have a cube."] }), "\n", _jsx(_components.p, { children: "NEXT: moving to flows / processes." }), "\n", _jsxs(_components.h2, { id: "is-there-room--need-for-a-simple-dataflow-library--2020-02-23-rufuspollock", children: [_jsx(_components.a, { className: "heading-link", href: "#is-there-room--need-for-a-simple-dataflow-library--2020-02-23-rufuspollock", children: _jsx(_components.svg, { xmlns: "http:www.w3.org/2000/svg", fill: "#ab2b65", viewBox: "0 0 20 20", className: "w-5 h-5", children: _jsx(_components.path, { fillRule: "evenodd", clipRule: "evenodd", d: "M9.493 2.853a.75.75 0 00-1.486-.205L7.545 6H4.198a.75.75 0 000 1.5h3.14l-.69 5H3.302a.75.75 0 000 1.5h3.14l-.435 3.148a.75.75 0 001.486.205L7.955 14h2.986l-.434 3.148a.75.75 0 001.486.205L12.456 14h3.346a.75.75 0 000-1.5h-3.14l.69-5h3.346a.75.75 0 000-1.5h-3.14l.435-3.147a.75.75 0 00-1.486-.205L12.045 6H9.059l.434-3.147zM8.852 7.5l-.69 5h2.986l.69-5H8.852z" }) }) }), "Is there room / need for a simple dataflow library? 2020-02-23 @rufuspollock"] }), "\n", _jsx(_components.p, { children: "Is there room / need for a simple dataflow library …?" }), "\n", _jsx(_components.p, { children: "What kind of library?" }), "\n", _jsxs(_components.ul, { children: ["\n", _jsx(_components.li, { children: "So Apache Beam /Google DataFlow is great … and it is pretty complex and designed for use out of the box in parallel etc" }), "\n", _jsx(_components.li, { children: "Apache Nifi: got a nice UI, Java and heavy duty." }), "\n", _jsx(_components.li, { children: "My own experience (even just now with key metrics) is i want something that will load and pipe data between processors" }), "\n"] }), "\n", _jsx(_components.p, { children: "Ideas" }), "\n", _jsxs(_components.ul, { children: ["\n", _jsx(_components.li, { children: "create-react-app for data flows: quickly scaffold data flows" }), "\n", _jsx(_components.li, { children: "What is the default runner" }), "\n"] })] }); } function MDXContent(props = {}) { const {wrapper: MDXLayout} = { ..._provideComponents(), ...props.components }; return MDXLayout ? _jsx(MDXLayout, { ...props, children: _jsx(_createMdxContent, { ...props }) }) : _createMdxContent(props); } return { default: MDXContent };