chunking_toy.exs

#!/usr/bin/env elixir

Registry.start_link(keys: :unique, name: M.Registry)

defmodule ChunkServer do
  use GenServer
  require Logger

  def get_chunk(key, n) do
    if pid = GenServer.whereis(name(key)) do
      {:ok, pid}
    else
      Logger.debug("server not started, starting")
      start(key: key)
    end

    GenServer.call(name(key), {:get_chunk, n})
  end

  def start(args) do
    IO.inspect(name(args[:key]))
    GenServer.start(__MODULE__, args, name: name(args[:key]))
  end

  def init(args) do
    {:ok, %{}, {:continue, {:fetch_file, args[:key]}}}
  end

  def handle_continue({:fetch_file, key}, state) do
    Logger.debug("fetching file")
    file = fetch_file(key)
    chunks = M.line_offset_lengths(file)

    state =
      state
      |> Map.put(:file, file)
      |> Map.put(:chunks, chunks)

    {:noreply, state}
  end

  def handle_call({:get_chunk, n}, _from, state) do
    file = state[:file]
    {:ok, f} = :file.open(file, [:read, :ram])
    {offset, length} = Enum.at(state[:chunks], n)
    reply = :file.pread(f, offset, length)

    {:reply, reply, state}
  end

  defp fetch_file(_key) do
    "a,b,c\n1,2,3\n4,5,6\n7,8,9\n10,11,12\n"
  end

  defp name(key) do
    {:via, Registry, {M.Registry, {__MODULE__, key}}}
  end
end

defmodule M do
  def line_offset_lengths(s) when is_binary(s) do
    s
    |> String.trim()
    |> do_line_offset_lengths(0, 0, [])
    |> Enum.reverse()
  end

  defp do_line_offset_lengths(<<>>, offset, length, offset_lengths) do
    [{offset, length} | offset_lengths]
  end

  defp do_line_offset_lengths(<<"\n", rest::binary>>, offset, length, offset_lengths) do
    do_line_offset_lengths(rest, offset + length + 1, 0, [{offset, length} | offset_lengths])
  end

  defp do_line_offset_lengths(<<_::binary-1, rest::binary>>, offset, length, offset_lengths) do
    do_line_offset_lengths(rest, offset, length + 1, offset_lengths)
  end
end

# s = "a,b,c\n1,2,3\n4,5,6\n7,8,9\n10,11,12\n"

# offset_lengths =
#   s
#   |> M.line_offset_lengths()
#   |> dbg

# {:ok, f} = :file.open(s, [:read, :ram])

# offset_lengths
# |> Enum.each(fn {offset, length} ->
#   {:ok, data} = :file.pread(f, offset, length) |> dbg
# end)

for n <- 0..4 do
  ChunkServer.get_chunk("myfile.csv", n)
  |> dbg
end