Brainstorm on an interface to read a directory of Markdown files

main
Jeremy Boles 2 months ago
parent 28ccf1e650
commit 546bbfe6f1
Signed by: jb
GPG Key ID: 7DE0508F61CB57DF
  1. 3
      config/config.exs
  2. 3
      lib/ann.ex
  3. 2
      lib/storala.ex
  4. 38
      lib/storala/data.ex
  5. 69
      lib/storala/markdown.ex
  6. 27
      lib/storala/sidecar.ex
  7. 16
      lib/storala/wiki.ex
  8. 18
      lib/storala/wiki/topic.ex
  9. 6
      mix.exs
  10. 7
      mix.lock

@ -1,2 +1,3 @@
import Config
import_config "#{config_env()}.exs"
import_config "#{config_env()}.exs"

@ -1,5 +1,2 @@
defmodule Ann do
@moduledoc """
Documentation for `Ann`.
"""
end

@ -0,0 +1,2 @@
defmodule Storala do
end

@ -0,0 +1,38 @@
defmodule Storala.Data do
@moduledoc """
Encapsulates accessing data extracted out of a document and normalize the data from a document.
"""
alias Storala.Sidecar
def cascade(%{frontmatter: nil, sidecar: sidecar}, key), do: get_sidecar(sidecar, key)
def cascade(%{frontmatter: frontmatter, sidecar: sidecar}, key) do
Map.get_lazy(frontmatter, key, fn -> get_sidecar(sidecar, key) end)
end
def normalize(list) when is_list(list), do: Enum.map(list, &normalize/1)
def normalize(map) do
map |> normalize(:coords)
end
defp normalize(%{"coords" => %{"lat" => lat, "lng" => lng}} = map, :coords) do
Map.update!(map, "coords", fn _ -> %Geo.Point{coordinates: {lng, lat}, srid: 4326} end)
end
defp normalize(map, _), do: map
defp get_sidecar(sidecar, key) do
Enum.find_value(sidecar, fn
%Sidecar{data: data, key: ^key} ->
data
%Sidecar{data: data} when is_list(data) ->
nil
%Sidecar{data: data} ->
Map.get(data, key, nil)
end)
end
end

@ -0,0 +1,69 @@
defmodule Storala.Markdown do
@moduledoc """
Encapsulates parsing a Markdown document, along with any extra data.
"""
alias Storala.{Data, Sidecar}
@fields content_html: nil, filename: nil, frontmatter: nil, links: [], sidecar: [], tags: []
defstruct @fields
defmacro __using__(fields) when is_list(fields) do
quote do
defstruct unquote(@fields) ++ unquote(fields)
end
end
def parse_file!(filename, include_sidebar_files \\ true) do
read(filename)
|> parse_frontmatter()
|> parse_hashtags()
|> parse_contents()
|> parse_sidebar_files(include_sidebar_files)
|> extract_internal_links()
end
defp collect_link({"a", [{"href", id}], _}), do: id
defp collect_tag([_, _, tag, _]), do: tag
@selector ~s<a:not([href^="/"]):not([href$="/"]):not([href^="https://"]):not([href^="http://"])>
def extract_internal_links(%__MODULE__{content_html: html} = struct) do
links = html |> Floki.parse_document!() |> Floki.find(@selector) |> Enum.map(&collect_link/1)
%{struct | links: links}
end
defp parse_contents({struct, contents}) do
options = %Earmark.Options{compact_output: false, smartypants: true, wikilinks: true}
%{struct | content_html: Earmark.as_html!(contents, options)}
end
@hashtag ~r/(^|\B)#(?![0-9_]+\b)([a-zA-Z0-9_]{1,30})(\b|\r)/
def parse_hashtags({struct, contents}) do
case Regex.scan(@hashtag, contents) do
[] ->
{struct, contents}
tags when is_list(tags) ->
{%{struct | tags: Enum.map(tags, &collect_tag/1)}, Regex.replace(@hashtag, contents, "")}
end
end
@frontmatter ~r/^---\n(.*)\n---\n([\s\S]+)/
defp parse_frontmatter({struct, contents}) do
case Regex.run(@frontmatter, contents) do
[_, yaml, contents] -> {%{struct | frontmatter: parse_yaml(yaml)}, contents}
_ -> {struct, contents}
end
end
defp parse_sidebar_files(%__MODULE__{filename: filename} = struct, true) do
%{struct | sidecar: filename |> Sidecar.for_file() |> Enum.map(&Sidecar.load/1)}
end
defp parse_sidebar_files(%__MODULE__{} = struct, _), do: struct
defp parse_yaml(yaml), do: yaml |> YamlElixir.read_from_string!() |> Data.normalize()
defp read(filename), do: {%__MODULE__{filename: filename}, File.read!(filename)}
end

@ -0,0 +1,27 @@
defmodule Storala.Sidecar do
@moduledoc """
Encapsulates getting a Markdown document's sidecar data out of is corresponding documents.
"""
defstruct [:data, :filename, :key]
alias Storala.Data
def for_file(filename) do
search = Path.dirname(filename) <> "/." <> Path.basename(filename, ".md") <> "**.yml"
Path.wildcard(search, match_dot: true)
end
def load(filename) do
%__MODULE__{data: parse_yaml(filename), filename: filename, key: key(filename)}
end
defp key(filename) do
case filename |> Path.basename(".yml") |> Path.extname() do
"." <> key -> key
"" -> nil
end
end
defp parse_yaml(filename), do: filename |> YamlElixir.read_from_file!() |> Data.normalize()
end

@ -0,0 +1,16 @@
defmodule Storala.Wiki do
alias Storala.Markdown
alias __MODULE__.Topic
use Markdown, id: nil, kind: "topic"
def all do
Path.wildcard(wiki_path() <> "/**/*.md")
|> Enum.map(&Markdown.parse_file!/1)
|> Enum.map(&Topic.new/1)
end
def get_topic(collection, id), do: collection |> Enum.find(&(&1.id == id))
defp wiki_path, do: Path.expand("~/Ann/wiki")
end

@ -0,0 +1,18 @@
defmodule Storala.Wiki.Topic do
alias Storala.{Data, Markdown}
use Markdown, id: nil, kind: "topic"
def get(%__MODULE__{} = topic, "slug"), do: Data.cascade(topic, "slug") |> get_slug(topic)
def get(%__MODULE__{} = topic, key), do: Data.cascade(topic, key)
def new(%Markdown{} = struct), do: struct |> Map.from_struct() |> new()
def new(map), do: struct(__MODULE__, map_fields(map))
defp get_slug(nil, %__MODULE__{id: id}), do: id
defp get_slug(slug, _), do: slug
defp map_fields(%{filename: filename} = map) do
Map.put(map, :id, Path.basename(filename, ".md"))
end
end

@ -23,7 +23,11 @@ defmodule Ann.MixProject do
defp deps do
[
{:bandit, "~> 0.5"},
{:plug, "~> 1.13"}
{:earmark, "~> 1.4.26"},
{:floki, "~> 0.33.1"},
{:geo, "~> 3.4"},
{:plug, "~> 1.13"},
{:yaml_elixir, "~> 2.9.0"}
]
end
end

@ -1,9 +1,16 @@
%{
"bandit": {:hex, :bandit, "0.5.0", "188c1d3ade5760eded7a08798df9e231c8f07e406505a913629dd217b6fa96a7", [:mix], [{:hpax, "~> 0.1.1", [hex: :hpax, repo: "hexpm", optional: false]}, {:plug, "~> 1.12", [hex: :plug, repo: "hexpm", optional: false]}, {:thousand_island, "~> 0.5.7", [hex: :thousand_island, repo: "hexpm", optional: false]}], "hexpm", "2e062f7e9b72e1bfba7bceecfb7566700a8fbe71f91dd7add6c71f584e67d445"},
"earmark": {:hex, :earmark, "1.4.26", "f0e3c3d5c278a6d448ad8c27ab0ecdec9c57a7710553138c56af220a6330a4fd", [:mix], [{:earmark_parser, "~> 1.4.26", [hex: :earmark_parser, repo: "hexpm", optional: false]}], "hexpm", "e1231882b56bece0692af33f0959f06c9cd580c2dc2ecb1dc9f16f2750fa78c5"},
"earmark_parser": {:hex, :earmark_parser, "1.4.26", "f4291134583f373c7d8755566122908eb9662df4c4b63caa66a0eabe06569b0a", [:mix], [], "hexpm", "48d460899f8a0c52c5470676611c01f64f3337bad0b26ddab43648428d94aabc"},
"floki": {:hex, :floki, "0.33.1", "f20f1eb471e726342b45ccb68edb9486729e7df94da403936ea94a794f072781", [:mix], [{:html_entities, "~> 0.5.0", [hex: :html_entities, repo: "hexpm", optional: false]}], "hexpm", "461035fd125f13fdf30f243c85a0b1e50afbec876cbf1ceefe6fddd2e6d712c6"},
"geo": {:hex, :geo, "3.4.3", "0ddf3f681993d32c397e5ef346e7b4b6f36f39ed138502429832fa4000ebb9d5", [:mix], [{:jason, "~> 1.2", [hex: :jason, repo: "hexpm", optional: true]}], "hexpm", "e23f2892e5437ec8b063cee1beccec89c58fd841ae11133304700235feb25552"},
"hpax": {:hex, :hpax, "0.1.1", "2396c313683ada39e98c20a75a82911592b47e5c24391363343bde74f82396ca", [:mix], [], "hexpm", "0ae7d5a0b04a8a60caf7a39fcf3ec476f35cc2cc16c05abea730d3ce6ac6c826"},
"html_entities": {:hex, :html_entities, "0.5.2", "9e47e70598da7de2a9ff6af8758399251db6dbb7eebe2b013f2bbd2515895c3c", [:mix], [], "hexpm", "c53ba390403485615623b9531e97696f076ed415e8d8058b1dbaa28181f4fdcc"},
"mime": {:hex, :mime, "2.0.2", "0b9e1a4c840eafb68d820b0e2158ef5c49385d17fb36855ac6e7e087d4b1dcc5", [:mix], [], "hexpm", "e6a3f76b4c277739e36c2e21a2c640778ba4c3846189d5ab19f97f126df5f9b7"},
"plug": {:hex, :plug, "1.13.6", "187beb6b67c6cec50503e940f0434ea4692b19384d47e5fdfd701e93cadb4cc2", [:mix], [{:mime, "~> 1.0 or ~> 2.0", [hex: :mime, repo: "hexpm", optional: false]}, {:plug_crypto, "~> 1.1.1 or ~> 1.2", [hex: :plug_crypto, repo: "hexpm", optional: false]}, {:telemetry, "~> 0.4.3 or ~> 1.0", [hex: :telemetry, repo: "hexpm", optional: false]}], "hexpm", "02b9c6b9955bce92c829f31d6284bf53c591ca63c4fb9ff81dfd0418667a34ff"},
"plug_crypto": {:hex, :plug_crypto, "1.2.2", "05654514ac717ff3a1843204b424477d9e60c143406aa94daf2274fdd280794d", [:mix], [], "hexpm", "87631c7ad914a5a445f0a3809f99b079113ae4ed4b867348dd9eec288cecb6db"},
"telemetry": {:hex, :telemetry, "1.1.0", "a589817034a27eab11144ad24d5c0f9fab1f58173274b1e9bae7074af9cbee51", [:rebar3], [], "hexpm", "b727b2a1f75614774cff2d7565b64d0dfa5bd52ba517f16543e6fc7efcc0df48"},
"thousand_island": {:hex, :thousand_island, "0.5.9", "b1b8403c93be0bd87944bd9b1bebac694041cc85cb7bd3e24ad6352c4ace0063", [:mix], [{:telemetry, "~> 1.0", [hex: :telemetry, repo: "hexpm", optional: false]}], "hexpm", "cd689ed867897ef0165b8e5495673c77cbee32e13b7b3cb253cf417b584a3218"},
"yamerl": {:hex, :yamerl, "0.10.0", "4ff81fee2f1f6a46f1700c0d880b24d193ddb74bd14ef42cb0bcf46e81ef2f8e", [:rebar3], [], "hexpm", "346adb2963f1051dc837a2364e4acf6eb7d80097c0f53cbdc3046ec8ec4b4e6e"},
"yaml_elixir": {:hex, :yaml_elixir, "2.9.0", "9a256da867b37b8d2c1ffd5d9de373a4fda77a32a45b452f1708508ba7bbcb53", [:mix], [{:yamerl, "~> 0.10", [hex: :yamerl, repo: "hexpm", optional: false]}], "hexpm", "0cb0e7d4c56f5e99a6253ed1a670ed0e39c13fc45a6da054033928607ac08dfc"},
}

Loading…
Cancel
Save