Giza++.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "name": "Giza++",
      "provenance": [],
      "collapsed_sections": [],
      "authorship_tag": "ABX9TyPYpAUqDjn7AtU7z7TEcShq",
      "include_colab_link": true
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    },
    "language_info": {
      "name": "python"
    }
  },
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "view-in-github",
        "colab_type": "text"
      },
      "source": [
        "<a href=\"https://colab.research.google.com/github/quocthang0507/VietnameseNaturalLanguageProcessing/blob/main/Giza%2B%2B.ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "xf_KgBv5EL0E"
      },
      "source": [
        "https://www.techie-knowledge.co.in/2017/04/installing-and-using-giza-in-ubuntu-for.html"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "GOHzgYuq8xjX",
        "outputId": "2e012ab0-08bb-4a56-f180-1f596e41ccaf"
      },
      "source": [
        "!git clone https://github.com/moses-smt/giza-pp.git"
      ],
      "execution_count": 1,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Cloning into 'giza-pp'...\n",
            "remote: Enumerating objects: 301, done.\u001b[K\n",
            "remote: Total 301 (delta 0), reused 0 (delta 0), pack-reused 301\u001b[K\n",
            "Receiving objects: 100% (301/301), 259.42 KiB | 14.41 MiB/s, done.\n",
            "Resolving deltas: 100% (207/207), done.\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "_rozeZ-J81V-",
        "outputId": "3578b170-2a0c-4f8a-c1bc-aef8f9a9eb0e"
      },
      "source": [
        "%cd giza-pp/"
      ],
      "execution_count": 2,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "/content/giza-pp\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "6i3-ZWBX9Eyy",
        "outputId": "c5bc5394-7f1a-4aeb-ebab-ca8661187e70"
      },
      "source": [
        "!make clean"
      ],
      "execution_count": 3,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "make -C GIZA++-v2 clean\n",
            "make[1]: Entering directory '/content/giza-pp/GIZA++-v2'\n",
            "rm -f norm//*.o debug//*.o vdebug//*.o profile//*.o optimized//*.o\n",
            "rm -rf norm/ debug/ vdebug/ profile/ optimized/\n",
            "rm -f snt2plain.out plain2snt.out snt2cooc.out GIZA++\n",
            "make[1]: Leaving directory '/content/giza-pp/GIZA++-v2'\n",
            "make -C mkcls-v2 clean\n",
            "make[1]: Entering directory '/content/giza-pp/mkcls-v2'\n",
            "rm -f *.o mkcls\n",
            "make[1]: Leaving directory '/content/giza-pp/mkcls-v2'\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "0AdjkxCS-HIQ",
        "outputId": "840abcb8-0c10-4fc2-8775-e6b22ee65baa"
      },
      "source": [
        "!make"
      ],
      "execution_count": 4,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "make -C GIZA++-v2\n",
            "make[1]: Entering directory '/content/giza-pp/GIZA++-v2'\n",
            "mkdir optimized/\n",
            "g++   -Wall -Wno-parentheses -O3 -funroll-loops -DNDEBUG -DWORDINDEX_WITH_4_BYTE -DBINARY_SEARCH_FOR_TTABLE  -c Parameter.cpp -o optimized/Parameter.o\n",
            "\u001b[01m\u001b[KParameter.cpp:\u001b[m\u001b[K In function ‘\u001b[01m\u001b[Kbool writeParameters(std::ofstream&, const ParSet&, int)\u001b[m\u001b[K’:\n",
            "\u001b[01m\u001b[KParameter.cpp:48:14:\u001b[m\u001b[K \u001b[01;35m\u001b[Kwarning: \u001b[m\u001b[Kignoring return value of ‘\u001b[01m\u001b[Kchar* getcwd(char*, size_t)\u001b[m\u001b[K’, declared with attribute warn_unused_result [\u001b[01;35m\u001b[K-Wunused-result\u001b[m\u001b[K]\n",
            "        \u001b[01;35m\u001b[Kgetcwd(path,1024)\u001b[m\u001b[K;\n",
            "        \u001b[01;35m\u001b[K~~~~~~^~~~~~~~~~~\u001b[m\u001b[K\n",
            "g++   -Wall -Wno-parentheses -O3 -funroll-loops -DNDEBUG -DWORDINDEX_WITH_4_BYTE -DBINARY_SEARCH_FOR_TTABLE  -c myassert.cpp -o optimized/myassert.o\n",
            "g++   -Wall -Wno-parentheses -O3 -funroll-loops -DNDEBUG -DWORDINDEX_WITH_4_BYTE -DBINARY_SEARCH_FOR_TTABLE  -c Perplexity.cpp -o optimized/Perplexity.o\n",
            "g++   -Wall -Wno-parentheses -O3 -funroll-loops -DNDEBUG -DWORDINDEX_WITH_4_BYTE -DBINARY_SEARCH_FOR_TTABLE  -c model1.cpp -o optimized/model1.o\n",
            "g++   -Wall -Wno-parentheses -O3 -funroll-loops -DNDEBUG -DWORDINDEX_WITH_4_BYTE -DBINARY_SEARCH_FOR_TTABLE  -c model2.cpp -o optimized/model2.o\n",
            "g++   -Wall -Wno-parentheses -O3 -funroll-loops -DNDEBUG -DWORDINDEX_WITH_4_BYTE -DBINARY_SEARCH_FOR_TTABLE  -c model3.cpp -o optimized/model3.o\n",
            "g++   -Wall -Wno-parentheses -O3 -funroll-loops -DNDEBUG -DWORDINDEX_WITH_4_BYTE -DBINARY_SEARCH_FOR_TTABLE  -c getSentence.cpp -o optimized/getSentence.o\n",
            "g++   -Wall -Wno-parentheses -O3 -funroll-loops -DNDEBUG -DWORDINDEX_WITH_4_BYTE -DBINARY_SEARCH_FOR_TTABLE  -c TTables.cpp -o optimized/TTables.o\n",
            "g++   -Wall -Wno-parentheses -O3 -funroll-loops -DNDEBUG -DWORDINDEX_WITH_4_BYTE -DBINARY_SEARCH_FOR_TTABLE  -c ATables.cpp -o optimized/ATables.o\n",
            "g++   -Wall -Wno-parentheses -O3 -funroll-loops -DNDEBUG -DWORDINDEX_WITH_4_BYTE -DBINARY_SEARCH_FOR_TTABLE  -c AlignTables.cpp -o optimized/AlignTables.o\n",
            "g++   -Wall -Wno-parentheses -O3 -funroll-loops -DNDEBUG -DWORDINDEX_WITH_4_BYTE -DBINARY_SEARCH_FOR_TTABLE  -c main.cpp -o optimized/main.o\n",
            "g++   -Wall -Wno-parentheses -O3 -funroll-loops -DNDEBUG -DWORDINDEX_WITH_4_BYTE -DBINARY_SEARCH_FOR_TTABLE  -c NTables.cpp -o optimized/NTables.o\n",
            "g++   -Wall -Wno-parentheses -O3 -funroll-loops -DNDEBUG -DWORDINDEX_WITH_4_BYTE -DBINARY_SEARCH_FOR_TTABLE  -c model2to3.cpp -o optimized/model2to3.o\n",
            "g++   -Wall -Wno-parentheses -O3 -funroll-loops -DNDEBUG -DWORDINDEX_WITH_4_BYTE -DBINARY_SEARCH_FOR_TTABLE  -c collCounts.cpp -o optimized/collCounts.o\n",
            "g++   -Wall -Wno-parentheses -O3 -funroll-loops -DNDEBUG -DWORDINDEX_WITH_4_BYTE -DBINARY_SEARCH_FOR_TTABLE  -c alignment.cpp -o optimized/alignment.o\n",
            "g++   -Wall -Wno-parentheses -O3 -funroll-loops -DNDEBUG -DWORDINDEX_WITH_4_BYTE -DBINARY_SEARCH_FOR_TTABLE  -c vocab.cpp -o optimized/vocab.o\n",
            "g++   -Wall -Wno-parentheses -O3 -funroll-loops -DNDEBUG -DWORDINDEX_WITH_4_BYTE -DBINARY_SEARCH_FOR_TTABLE  -c MoveSwapMatrix.cpp -o optimized/MoveSwapMatrix.o\n",
            "g++   -Wall -Wno-parentheses -O3 -funroll-loops -DNDEBUG -DWORDINDEX_WITH_4_BYTE -DBINARY_SEARCH_FOR_TTABLE  -c transpair_model3.cpp -o optimized/transpair_model3.o\n",
            "g++   -Wall -Wno-parentheses -O3 -funroll-loops -DNDEBUG -DWORDINDEX_WITH_4_BYTE -DBINARY_SEARCH_FOR_TTABLE  -c transpair_model5.cpp -o optimized/transpair_model5.o\n",
            "g++   -Wall -Wno-parentheses -O3 -funroll-loops -DNDEBUG -DWORDINDEX_WITH_4_BYTE -DBINARY_SEARCH_FOR_TTABLE  -c transpair_model4.cpp -o optimized/transpair_model4.o\n",
            "g++   -Wall -Wno-parentheses -O3 -funroll-loops -DNDEBUG -DWORDINDEX_WITH_4_BYTE -DBINARY_SEARCH_FOR_TTABLE  -c utility.cpp -o optimized/utility.o\n",
            "g++   -Wall -Wno-parentheses -O3 -funroll-loops -DNDEBUG -DWORDINDEX_WITH_4_BYTE -DBINARY_SEARCH_FOR_TTABLE  -c parse.cpp -o optimized/parse.o\n",
            "g++   -Wall -Wno-parentheses -O3 -funroll-loops -DNDEBUG -DWORDINDEX_WITH_4_BYTE -DBINARY_SEARCH_FOR_TTABLE  -c reports.cpp -o optimized/reports.o\n",
            "g++   -Wall -Wno-parentheses -O3 -funroll-loops -DNDEBUG -DWORDINDEX_WITH_4_BYTE -DBINARY_SEARCH_FOR_TTABLE  -c model3_viterbi.cpp -o optimized/model3_viterbi.o\n",
            "g++   -Wall -Wno-parentheses -O3 -funroll-loops -DNDEBUG -DWORDINDEX_WITH_4_BYTE -DBINARY_SEARCH_FOR_TTABLE  -c model3_viterbi_with_tricks.cpp -o optimized/model3_viterbi_with_tricks.o\n",
            "g++   -Wall -Wno-parentheses -O3 -funroll-loops -DNDEBUG -DWORDINDEX_WITH_4_BYTE -DBINARY_SEARCH_FOR_TTABLE  -c Dictionary.cpp -o optimized/Dictionary.o\n",
            "g++   -Wall -Wno-parentheses -O3 -funroll-loops -DNDEBUG -DWORDINDEX_WITH_4_BYTE -DBINARY_SEARCH_FOR_TTABLE  -c model345-peg.cpp -o optimized/model345-peg.o\n",
            "g++   -Wall -Wno-parentheses -O3 -funroll-loops -DNDEBUG -DWORDINDEX_WITH_4_BYTE -DBINARY_SEARCH_FOR_TTABLE  -c hmm.cpp -o optimized/hmm.o\n",
            "g++   -Wall -Wno-parentheses -O3 -funroll-loops -DNDEBUG -DWORDINDEX_WITH_4_BYTE -DBINARY_SEARCH_FOR_TTABLE  -c HMMTables.cpp -o optimized/HMMTables.o\n",
            "g++   -Wall -Wno-parentheses -O3 -funroll-loops -DNDEBUG -DWORDINDEX_WITH_4_BYTE -DBINARY_SEARCH_FOR_TTABLE  -c ForwardBackward.cpp -o optimized/ForwardBackward.o\n",
            "g++  -Wall -Wno-parentheses -O3 -funroll-loops -DNDEBUG -DWORDINDEX_WITH_4_BYTE -DBINARY_SEARCH_FOR_TTABLE optimized/Parameter.o optimized/myassert.o optimized/Perplexity.o optimized/model1.o optimized/model2.o optimized/model3.o optimized/getSentence.o optimized/TTables.o optimized/ATables.o optimized/AlignTables.o optimized/main.o optimized/NTables.o optimized/model2to3.o optimized/collCounts.o optimized/alignment.o optimized/vocab.o optimized/MoveSwapMatrix.o optimized/transpair_model3.o optimized/transpair_model5.o optimized/transpair_model4.o optimized/utility.o optimized/parse.o optimized/reports.o optimized/model3_viterbi.o optimized/model3_viterbi_with_tricks.o optimized/Dictionary.o optimized/model345-peg.o optimized/hmm.o optimized/HMMTables.o optimized/ForwardBackward.o  -o GIZA++\n",
            "g++  -O3 -W -Wall snt2plain.cpp -o snt2plain.out\n",
            "g++  -O3 -W -Wall plain2snt.cpp -o plain2snt.out\n",
            "g++  -O3 -g -W -Wall snt2cooc.cpp -o snt2cooc.out\n",
            "make[1]: Leaving directory '/content/giza-pp/GIZA++-v2'\n",
            "make -C mkcls-v2\n",
            "make[1]: Entering directory '/content/giza-pp/mkcls-v2'\n",
            "g++ -Wall -W -DNDEBUG -O3 -funroll-loops -c GDAOptimization.cpp -o GDAOptimization.o\n",
            "g++ -Wall -W -DNDEBUG -O3 -funroll-loops -c HCOptimization.cpp -o HCOptimization.o\n",
            "g++ -Wall -W -DNDEBUG -O3 -funroll-loops -c Problem.cpp -o Problem.o\n",
            "g++ -Wall -W -DNDEBUG -O3 -funroll-loops -c IterOptimization.cpp -o IterOptimization.o\n",
            "g++ -Wall -W -DNDEBUG -O3 -funroll-loops -c ProblemTest.cpp -o ProblemTest.o\n",
            "g++ -Wall -W -DNDEBUG -O3 -funroll-loops -c RRTOptimization.cpp -o RRTOptimization.o\n",
            "g++ -Wall -W -DNDEBUG -O3 -funroll-loops -c MYOptimization.cpp -o MYOptimization.o\n",
            "g++ -Wall -W -DNDEBUG -O3 -funroll-loops -c SAOptimization.cpp -o SAOptimization.o\n",
            "g++ -Wall -W -DNDEBUG -O3 -funroll-loops -c TAOptimization.cpp -o TAOptimization.o\n",
            "g++ -Wall -W -DNDEBUG -O3 -funroll-loops -c Optimization.cpp -o Optimization.o\n",
            "g++ -Wall -W -DNDEBUG -O3 -funroll-loops -c KategProblemTest.cpp -o KategProblemTest.o\n",
            "g++ -Wall -W -DNDEBUG -O3 -funroll-loops -c KategProblemKBC.cpp -o KategProblemKBC.o\n",
            "g++ -Wall -W -DNDEBUG -O3 -funroll-loops -c KategProblemWBC.cpp -o KategProblemWBC.o\n",
            "g++ -Wall -W -DNDEBUG -O3 -funroll-loops -c KategProblem.cpp -o KategProblem.o\n",
            "\u001b[01m\u001b[KKategProblem.cpp:\u001b[m\u001b[K In member function ‘\u001b[01m\u001b[Kvirtual int KategProblem::_change(ProblemChange**)\u001b[m\u001b[K’:\n",
            "\u001b[01m\u001b[KKategProblem.cpp:484:10:\u001b[m\u001b[K \u001b[01;35m\u001b[Kwarning: \u001b[m\u001b[Kthis statement may fall through [\u001b[01;35m\u001b[K-Wimplicit-fallthrough=\u001b[m\u001b[K]\n",
            "       \u001b[01;35m\u001b[Kkat=randomInt(katFreq.nKats-2)+2\u001b[m\u001b[K;\n",
            "       \u001b[01;35m\u001b[K~~~^~~~~~~~~~~~~~~~~~~~~~~~~~~~~\u001b[m\u001b[K\n",
            "\u001b[01m\u001b[KKategProblem.cpp:486:5:\u001b[m\u001b[K \u001b[01;36m\u001b[Knote: \u001b[m\u001b[Khere\n",
            "     \u001b[01;36m\u001b[Kcase\u001b[m\u001b[K K_DET:\n",
            "     \u001b[01;36m\u001b[K^~~~\u001b[m\u001b[K\n",
            "g++ -Wall -W -DNDEBUG -O3 -funroll-loops -c StatVar.cpp -o StatVar.o\n",
            "g++ -Wall -W -DNDEBUG -O3 -funroll-loops -c general.cpp -o general.o\n",
            "g++ -Wall -W -DNDEBUG -O3 -funroll-loops -c mkcls.cpp -o mkcls.o\n",
            "g++ -Wall -W -DNDEBUG -O3 -funroll-loops -o mkcls GDAOptimization.o HCOptimization.o Problem.o IterOptimization.o ProblemTest.o RRTOptimization.o MYOptimization.o SAOptimization.o TAOptimization.o Optimization.o KategProblemTest.o KategProblemKBC.o KategProblemWBC.o KategProblem.o StatVar.o general.o mkcls.o \n",
            "make[1]: Leaving directory '/content/giza-pp/mkcls-v2'\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "isOn9LRW_fD-",
        "outputId": "3062d043-96f1-453f-8334-eef5a3cd7de6"
      },
      "source": [
        "!/content/giza-pp/GIZA++-v2/plain2snt.out /content/vi.src.txt /content/kho.tgt.txt"
      ],
      "execution_count": 6,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "w1:/content/vi.src w2:/content/kho.tgt\n",
            "/content/vi.src -> vi.src\n",
            "/content/kho.tgt -> kho.tgt\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "We3baiGxGcsb",
        "outputId": "be7e7623-d05a-4dc6-c2d0-d347f7cdcce3"
      },
      "source": [
        "!/content/giza-pp/mkcls-v2/mkcls"
      ],
      "execution_count": 16,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "ERROR: can not open file train.\n",
            "Error: Could not read the file 'train'.\n",
            "mkcls - a program for making word classes: Usage: \n",
            " mkcls [-nnum] [-ptrain] [-Vfile] opt\n",
            "-V output classes (Default: no file)\n",
            "-n number of optimization runs (Default: 1); larger number => better results\n",
            "-p filename of training corpus (Default: 'train')\n",
            "Example:\n",
            " mkcls -c80 -n10 -pin -Vout opt\n",
            " (generates 80 classes for the corpus 'in' and writes the classes in 'out')\n",
            "Literature: \n",
            " Franz Josef Och: �Maximum-Likelihood-Sch�tzung von Wortkategorien mit Verfahren\n",
            " der kombinatorischen Optimierung� Studienarbeit, Universit�t Erlangen-N�rnberg,\n",
            " Germany,1995. \n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "bJlgS3tk_mpc",
        "outputId": "c53d4686-77ff-45f8-8648-3dfb4b586227"
      },
      "source": [
        "!/content/giza-pp/mkcls-v2/mkcls -p/content/vi.src.txt -V/content/vi.vcb.classes"
      ],
      "execution_count": 17,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "\n",
            "***** 1 runs. (algorithm:TA)*****\n",
            ";KategProblem:cats: 100   words: 17365\n",
            "\n",
            "start-costs: MEAN: 4.01612e+06 (4.01612e+06-4.01612e+06)  SIGMA:0   \n",
            "  end-costs: MEAN: 3.64866e+06 (3.64866e+06-3.64866e+06)  SIGMA:0   \n",
            "   start-pp: MEAN: 1351.59 (1351.59-1351.59)  SIGMA:0   \n",
            "     end-pp: MEAN: 433.864 (433.864-433.864)  SIGMA:0   \n",
            " iterations: MEAN: 457952 (457952-457952)  SIGMA:0   \n",
            "       time: MEAN: 20.991 (20.991-20.991)  SIGMA:0   \n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "Yycy0iNY_p76",
        "outputId": "1dbf6ba1-2c54-4855-873a-dce4efd15e13"
      },
      "source": [
        "!/content/giza-pp/mkcls-v2/mkcls -p/content/kho.tgt.txt -V/content/kho.vcb.classes"
      ],
      "execution_count": 18,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "\n",
            "***** 1 runs. (algorithm:TA)*****\n",
            ";KategProblem:cats: 100   words: 20261\n",
            "\n",
            "start-costs: MEAN: 4.44376e+06 (4.44376e+06-4.44376e+06)  SIGMA:0   \n",
            "  end-costs: MEAN: 4.04493e+06 (4.04493e+06-4.04493e+06)  SIGMA:0   \n",
            "   start-pp: MEAN: 836.823 (836.823-836.823)  SIGMA:0   \n",
            "     end-pp: MEAN: 273.064 (273.064-273.064)  SIGMA:0   \n",
            " iterations: MEAN: 516038 (516038-516038)  SIGMA:0   \n",
            "       time: MEAN: 22.1708 (22.1708-22.1708)  SIGMA:0   \n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "6i3ajiFx_sbJ"
      },
      "source": [
        "!mkdir /content/output"
      ],
      "execution_count": 7,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "asgAnNPVBnLp",
        "outputId": "b86c6f41-d49d-4d4d-c97a-3c692eaacb34"
      },
      "source": [
        "!/content/giza-pp/GIZA++-v2/GIZA++"
      ],
      "execution_count": 8,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Usage:\n",
            "\n",
            "./GIZA++ <config_file> [options]\n",
            "\n",
            "Options (these override parameters set in the config file):\n",
            "\n",
            "\t--v \t\t print verbose message, Warning this is not very descriptive and not systematic.\n",
            "\t--NODUMPS \t Do not write any files to disk (This will over write dump frequency options).\n",
            "\t--h[elp]\t\tprint this help\n",
            "\t--p\t\tUse pegging when generating alignments for Model3 training.  (Default NO PEGGING)\n",
            "\t--st\t\tto use a fixed ditribution for the fertility parameters when tranfering from model 2 to model 3 (Default complicated estimation)\n",
            "general parameters:\n",
            "-------------------\n",
            "ml = 101  (maximum sentence length)\n",
            "\n",
            "No. of iterations:\n",
            "-------------------\n",
            "hmmiterations = 5  (mh)\n",
            "model1iterations = 5  (number of iterations for Model 1)\n",
            "model2iterations = 0  (number of iterations for Model 2)\n",
            "model3iterations = 5  (number of iterations for Model 3)\n",
            "model4iterations = 5  (number of iterations for Model 4)\n",
            "model5iterations = 0  (number of iterations for Model 5)\n",
            "model6iterations = 0  (number of iterations for Model 6)\n",
            "\n",
            "parameter for various heuristics in GIZA++ for efficient training:\n",
            "------------------------------------------------------------------\n",
            "countincreasecutoff = 1e-06  (Counts increment cutoff threshold)\n",
            "countincreasecutoffal = 1e-05  (Counts increment cutoff threshold for alignments in training of fertility models)\n",
            "mincountincrease = 1e-07  (minimal count increase)\n",
            "peggedcutoff = 0.03  (relative cutoff probability for alignment-centers in pegging)\n",
            "probcutoff = 1e-07  (Probability cutoff threshold for lexicon probabilities)\n",
            "probsmooth = 1e-07  (probability smoothing (floor) value )\n",
            "\n",
            "parameters for describing the type and amount of output:\n",
            "-----------------------------------------------------------\n",
            "compactalignmentformat = 0  (0: detailled alignment format, 1: compact alignment format )\n",
            "hmmdumpfrequency = 0  (dump frequency of HMM)\n",
            "l =   (log file name)\n",
            "log = 0  (0: no logfile; 1: logfile)\n",
            "model1dumpfrequency = 0  (dump frequency of Model 1)\n",
            "model2dumpfrequency = 0  (dump frequency of Model 2)\n",
            "model345dumpfrequency = 0  (dump frequency of Model 3/4/5)\n",
            "nbestalignments = 0  (for printing the n best alignments)\n",
            "nodumps = 0  (1: do not write any files)\n",
            "o =   (output file prefix)\n",
            "onlyaldumps = 0  (1: do not write any files)\n",
            "outputpath =   (output path)\n",
            "transferdumpfrequency = 0  (output: dump of transfer from Model 2 to 3)\n",
            "verbose = 0  (0: not verbose; 1: verbose)\n",
            "verbosesentence = -10  (number of sentence for which a lot of information should be printed (negative: no output))\n",
            "\n",
            "parameters describing input files:\n",
            "----------------------------------\n",
            "c =   (training corpus file name)\n",
            "d =   (dictionary file name)\n",
            "s =   (source vocabulary file name)\n",
            "t =   (target vocabulary file name)\n",
            "tc =   (test corpus file name)\n",
            "\n",
            "smoothing parameters:\n",
            "---------------------\n",
            "emalsmooth = 0.2  (f-b-trn: smoothing factor for HMM alignment model (can be ignored by -emSmoothHMM))\n",
            "model23smoothfactor = 0  (smoothing parameter for IBM-2/3 (interpolation with constant))\n",
            "model4smoothfactor = 0.2  (smooting parameter for alignment probabilities in Model 4)\n",
            "model5smoothfactor = 0.1  (smooting parameter for distortion probabilities in Model 5 (linear interpolation with constant))\n",
            "nsmooth = 64  (smoothing for fertility parameters (good value: 64): weight for wordlength-dependent fertility parameters)\n",
            "nsmoothgeneral = 0  (smoothing for fertility parameters (default: 0): weight for word-independent fertility parameters)\n",
            "\n",
            "parameters modifying the models:\n",
            "--------------------------------\n",
            "compactadtable = 1  (1: only 3-dimensional alignment table for IBM-2 and IBM-3)\n",
            "deficientdistortionforemptyword = 0  (0: IBM-3/IBM-4 as described in (Brown et al. 1993); 1: distortion model of empty word is deficient; 2: distoriton model of empty word is deficient (differently); setting this parameter also helps to avoid that during IBM-3 and IBM-4 training too many words are aligned with the empty word)\n",
            "depm4 = 76  (d_{=1}: &1:l, &2:m, &4:F, &8:E, d_{>1}&16:l, &32:m, &64:F, &128:E)\n",
            "depm5 = 68  (d_{=1}: &1:l, &2:m, &4:F, &8:E, d_{>1}&16:l, &32:m, &64:F, &128:E)\n",
            "emalignmentdependencies = 2  (lextrain: dependencies in the HMM alignment model.  &1: sentence length; &2: previous class; &4: previous position;  &8: French position; &16: French class)\n",
            "emprobforempty = 0.4  (f-b-trn: probability for empty word)\n",
            "\n",
            "parameters modifying the EM-algorithm:\n",
            "--------------------------------------\n",
            "m5p0 = -1  (fixed value for parameter p_0 in IBM-5 (if negative then it is determined in training))\n",
            "manlexfactor1 = 0  ()\n",
            "manlexfactor2 = 0  ()\n",
            "manlexmaxmultiplicity = 20  ()\n",
            "maxfertility = 10  (maximal fertility for fertility models)\n",
            "p0 = -1  (fixed value for parameter p_0 in IBM-3/4 (if negative then it is determined in training))\n",
            "pegging = 0  (0: no pegging; 1: do pegging)\n",
            "\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "Mawe3sgQEW_a",
        "outputId": "bb408d30-6c7f-4d6a-8294-9957380f3a14"
      },
      "source": [
        "!/content/giza-pp/GIZA++-v2/snt2cooc.out '/content/vi.src.vcb' '/content/kho.tgt.vcb' '/content/vi.src_kho.tgt.snt' > '/content/src_tgt.cooc'"
      ],
      "execution_count": 11,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "line 1000\n",
            "line 2000\n",
            "line 3000\n",
            "line 4000\n",
            "line 5000\n",
            "line 6000\n",
            "line 7000\n",
            "line 8000\n",
            "line 9000\n",
            "line 10000\n",
            "line 11000\n",
            "line 12000\n",
            "line 13000\n",
            "END.\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "MTVD8ZGS_w5v"
      },
      "source": [
        "!/content/giza-pp/GIZA++-v2/GIZA++ -S '/content/vi.src.vcb' -T '/content/kho.tgt.vcb' -C '/content/vi.src_kho.tgt.snt' -CoocurrenceFile '/content/src_tgt.cooc' -o Result -outputpath '/content/output'"
      ],
      "execution_count": 22,
      "outputs": []
    }
  ]
}