wip

lib/philomena/data_exports/aggregator.ex

@@ -0,0 +1,161 @@
+defmodule Philomena.DataExports.Aggregator do
+  @moduledoc """
+  Data generation module for data export logic.
+  """
+
+  import Ecto.Query
+  alias Philomena.Repo
+
+  # Direct PII
+  alias Philomena.Donations.Donation
+  alias Philomena.UserFingerprints.UserFingerprint
+  alias Philomena.UserIps.UserIp
+  alias Philomena.UserNameChanges.UserNameChange
+  alias Philomena.Users.User
+
+  # UGC for export
+  alias Philomena.ArtistLinks.ArtistLink
+  alias Philomena.Badges.Award
+  alias Philomena.Comments.Comment
+  alias Philomena.Commissions.Commission
+  alias Philomena.DnpEntries.DnpEntry
+  alias Philomena.DuplicateReports.DuplicateReport
+  alias Philomena.Filters.Filter
+  alias Philomena.ImageFaves.ImageFave
+  alias Philomena.ImageHides.ImageHide
+  alias Philomena.ImageVotes.ImageVote
+  alias Philomena.Images.Image
+  alias Philomena.PollVotes.PollVote
+  alias Philomena.Posts.Post
+  alias Philomena.Reports.Report
+  alias Philomena.SourceChanges.SourceChange
+  alias Philomena.TagChanges.TagChange
+  alias Philomena.Topics.Topic
+  alias Philomena.Bans.User, as: UserBan
+
+  # Direct UGC from form submission
+  @user_columns [
+    :created_at,
+    :name,
+    :email,
+    :description,
+    :current_filter_id,
+    :spoiler_type,
+    :theme,
+    :images_per_page,
+    :show_large_thumbnails,
+    :show_sidebar_and_watched_images,
+    :fancy_tag_field_on_upload,
+    :fancy_tag_field_on_edit,
+    :fancy_tag_field_in_settings,
+    :autorefresh_by_default,
+    :anonymous_by_default,
+    :comments_newest_first,
+    :comments_always_jump_to_last,
+    :comments_per_page,
+    :watch_on_reply,
+    :watch_on_new_topic,
+    :watch_on_upload,
+    :messages_newest_first,
+    :serve_webm,
+    :no_spoilered_in_watched,
+    :watched_images_query_str,
+    :watched_images_exclude_str,
+    :use_centered_layout,
+    :personal_title,
+    :hide_vote_counts,
+    :scale_large_images
+  ]
+
+  # All these also have created_at and are selected by user_id
+  @indirect_columns [
+    {Donation, [:email, :amount, :fee, :note]},
+    {UserFingerprint, [:fingerprint, :uses, :updated_at]},
+    {UserIp, [:ip, :uses, :updated_at]},
+    {UserNameChange, [:name]},
+    {ArtistLink, [:aasm_state, :uri, :public, :tag_id]},
+    {Award, [:label, :badge_name, :badge_id]},
+    {Comment,
+     [
+       :ip,
+       :fingerprint,
+       :user_agent,
+       :referrer,
+       :anonymous,
+       :image_id,
+       :edited_at,
+       :edit_reason,
+       :body
+     ]},
+    {Commission,
+     [:open, :sheet_image_id, :categories, :information, :contact, :will_create, :will_not_create]},
+    {DnpEntry, [:tag_id, :aasm_state, :dnp_type, :hide_reason, :feedback, :reason, :instructions],
+     :requesting_user_id},
+    {DuplicateReport, [:reason, :image_id, :duplicate_of_image_id]},
+    {Filter,
+     [
+       :name,
+       :description,
+       :public,
+       :hidden_complex_str,
+       :spoilered_complex_str,
+       :hidden_tag_ids,
+       :spoilered_tag_ids
+     ]},
+    {ImageFave, [:image_id]},
+    {ImageHide, [:image_id]},
+    {ImageVote, [:image_id, :up]},
+    {Image, [:ip, :fingerprint, :user_agent, :referrer, :anonymous, :description]},
+    {PollVote, [:rank, :poll_option_id]},
+    {Post,
+     [:ip, :fingerprint, :user_agent, :referrer, :anonymous, :edited_at, :edit_reason, :body]},
+    {Report,
+     [:ip, :fingerprint, :user_agent, :referrer, :reason, :reportable_id, :reportable_type]},
+    {SourceChange, [:ip, :fingerprint, :user_agent, :referrer, :image_id, :added, :value]},
+    {TagChange,
+     [:ip, :fingerprint, :user_agent, :referrer, :image_id, :added, :tag_id, :tag_name_cache]},
+    {Topic, [:title, :anonymous, :forum_id]},
+    {UserBan, [:reason, :generated_ban_id]}
+  ]
+
+  @doc """
+  Get all of the export data for the given user.
+  """
+  def get_for_user(user_id) do
+    [select_user(user_id)] ++ select_indirect(user_id)
+  end
+
+  defp select_user(user_id) do
+    select_schema_by_key(user_id, User, @user_columns, :id)
+  end
+
+  defp select_indirect(user_id) do
+    Enum.map(@indirect_columns, fn
+      {schema_name, columns} ->
+        select_schema_by_key(user_id, schema_name, columns, :user_id)
+
+      {schema_name, columns, key_column} ->
+        select_schema_by_key(user_id, schema_name, columns, key_column)
+    end)
+  end
+
+  defp select_schema_by_key(user_id, schema_name, columns, key_column) do
+    table_name = schema_name.__schema__(:source)
+    columns = [:created_at] ++ columns
+
+    {table_name,
+     schema_name
+     |> where([s], field(s, ^key_column) == ^user_id)
+     |> select([s], map(s, ^columns))
+     |> Repo.all()
+     |> stringify_results()}
+  end
+
+  defp stringify_results(list_of_maps) do
+    Enum.map(list_of_maps, fn map ->
+      Map.new(map, fn {k, v} ->
+        {k, to_string(v)}
+      end)
+    end)
+  end
+end

lib/philomena/data_exports/zip_generator.ex

@@ -0,0 +1,25 @@
+defmodule Philomena.DataExports.ZipGenerator do
+  @moduledoc """
+  ZIP file generator for an export.
+  """
+
+  @doc """
+  Generate the ZIP file for the given aggregate data.
+
+  Expects a list of 2-tuples, with the first element being the name of the JSON
+  file to generate, and the second element being a list of maps to store.
+  """
+  def generate(data) do
+    raw_contents =
+      Enum.map(data, fn {k, v} ->
+        file_name = ~c"#{k}.json"
+        file_data = IO.iodata_to_binary(:json.encode(v))
+
+        {file_name, file_data}
+      end)
+
+    {:ok, {_filename, bytes}} = :zip.zip(~c"export.zip", raw_contents, [:memory])
+
+    bytes
+  end
+end