00_import_econstor_data.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "name": "import-econstor-data",
      "provenance": [],
      "collapsed_sections": [],
      "toc_visible": true
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    }
  },
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "ZZ5qbRBannWk"
      },
      "source": [
        "##Preparing the Project Directory\r\n",
        "Mounting GoogleDrive, creating a project folder if it does not exist yet, and setting the working directory to the project folder"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "1KvGitkLf9Gj",
        "outputId": "9ff297ec-e4a0-4979-a112-44f9f099d22d"
      },
      "source": [
        "from google.colab import drive\r\n",
        "import os\r\n",
        "\r\n",
        "drive.mount('/content/gdrive')  # Mounting GoogleDrive to the content folder\r\n",
        "\r\n",
        "project_dir = 'NLP_scientific-text-generation'\r\n",
        "if not os.path.exists('/content/gdrive/MyDrive/'+project_dir):  # Create a project folder if it does not exist yet\r\n",
        "    os.makedirs('/content/gdrive/MyDrive/'+project_dir)\r\n",
        "os.chdir('/content/gdrive/MyDrive/'+project_dir)  # Changing the working directory to the project folder on GoogleDrive"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Mounted at /content/gdrive\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "ZZEVYvpyoJSp"
      },
      "source": [
        "##Downloading and Unpacking the Data"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "nREJelVG5rnw"
      },
      "source": [
        "# If necessary the following code can be used to delete specific folders and the files included\r\n",
        "# USE WITH CARE\r\n",
        "\r\n",
        "#import shutil\r\n",
        "#shutil.rmtree('econstor-cc-by-4.0-txt')\r\n"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "SSHxLjRQ4SZS"
      },
      "source": [
        "####Function to download and unpack the data"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "xwVOefogw04B"
      },
      "source": [
        "import urllib.request\r\n",
        "import tarfile\r\n",
        "\r\n",
        "def getData (dataset):\r\n",
        "    # Connect download stream as tar file object\r\n",
        "    url = \"https://www.econstor.eu/ki-hackathon/\" + dataset\r\n",
        "    ftpstream = urllib.request.urlopen(url)\r\n",
        "    tf = tarfile.open(fileobj=ftpstream, mode=\"r|gz\")\r\n",
        "    # Extract files (files will be extracted into a subfolder included in the zip file)\r\n",
        "    tf.extractall()\r\n"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "3kf-PBcq4C-h"
      },
      "source": [
        "####Getting meta data in JSON format"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "l8Ge36XI4CGR"
      },
      "source": [
        "getData('econstor-cc-by-4.0-json.tgz')"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "2UgOX2yU3yAH"
      },
      "source": [
        "####Getting full text papers in ASCII"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "8L0AtZT_3uk5"
      },
      "source": [
        "getData('econstor-cc-by-4.0-txt.tgz')"
      ],
      "execution_count": null,
      "outputs": []
    }
  ]
}