autofreezeBert.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "#freeze_intermediate_e2e_lr.py\n",
    "from __future__ import absolute_import\n",
    "from __future__ import division\n",
    "from __future__ import print_function\n",
    "\n",
    "import csv\n",
    "import os\n",
    "import logging\n",
    "import argparse\n",
    "import random\n",
    "from tqdm import tqdm, trange\n",
    "import pandas as pd\n",
    "import math\n",
    "import numpy as np\n",
    "import torch\n",
    "from torch.utils.data import TensorDataset, DataLoader\n",
    "from torch.utils.data.sampler import RandomSampler, SequentialSampler, SubsetRandomSampler\n",
    "from torch.utils.data.distributed import DistributedSampler\n",
    "import time\n",
    "import tokenization\n",
    "from modeling_auto_freeze import BertConfig, BertForSequenceClassification\n",
    "from optimization_lr import BERTAdam\n",
    "import json\n",
    "import statistics \n",
    "from utils2 import *#autofreeze的utiles\n",
    "\n",
    "\n",
    "logging.basicConfig(format = '%(asctime)s - %(levelname)s - %(name)s -   %(message)s', \n",
    "                    datefmt = '%m/%d/%Y %H:%M:%S',\n",
    "                    level = logging.INFO)\n",
    "logger = logging.getLogger(__name__)\n",
    "\n",
    "\n",
    "def set_seed(seed):\n",
    "    random.seed(seed)\n",
    "    torch.manual_seed(seed)\n",
    "    torch.cuda.manual_seed_all(seed)\n",
    "    np.random.seed(seed)\n",
    "    os.environ['PYTHONHASHSEED'] = str(seed)\n",
    "    #TODO: Do we need deterministic in cudnn ? Double check\n",
    "    torch.backends.cudnn.deterministic = True\n",
    "    torch.backends.cudnn.benchmark = False\n",
    "    print (\"Seeded everything\")\n",
    "\n",
    "\n",
    "global_step = 0 \n",
    "nb_tr_examples = 0\n",
    "nb_tr_steps = 0\n",
    "tr_loss = 0         \n",
    "\n",
    "def eval_model(model, args, eval_dataloader, device, epoch, tr_loss, nb_tr_steps, eval_step):\n",
    "    model.eval()\n",
    "    eval_step = int(eval_step)\n",
    "    eval_loss, eval_accuracy = 0, 0\n",
    "    nb_eval_steps, nb_eval_examples = 0, 0\n",
    "    with open(os.path.join(args.output_dir, \"results_ep_\"+str(epoch)+\".txt\"),\"w\") as f:\n",
    "        for input_ids, input_mask, segment_ids, label_ids in eval_dataloader:\n",
    "            input_ids = input_ids.to(device)\n",
    "            input_mask = input_mask.to(device)\n",
    "            segment_ids = segment_ids.to(device)\n",
    "            label_ids = label_ids.to(device)\n",
    "\n",
    "            with torch.no_grad():\n",
    "                (tmp_eval_loss, logits), _ = model(input_ids, segment_ids, input_mask, label_ids)\n",
    "\n",
    "            logits = logits.detach().cpu().numpy()\n",
    "            label_ids = label_ids.to('cpu').numpy()\n",
    "            outputs = np.argmax(logits, axis=1)\n",
    "            for output in outputs:\n",
    "                f.write(str(output)+\"\\n\")\n",
    "            tmp_eval_accuracy=np.sum(outputs == label_ids)\n",
    "\n",
    "            eval_loss += tmp_eval_loss.mean().item()\n",
    "            eval_accuracy += tmp_eval_accuracy\n",
    "\n",
    "            nb_eval_examples += input_ids.size(0)\n",
    "            nb_eval_steps += 1\n",
    "\n",
    "    eval_loss = eval_loss / nb_eval_steps\n",
    "    eval_accuracy = eval_accuracy / nb_eval_examples\n",
    "\n",
    "    result = {'eval_loss': eval_loss,\n",
    "              'eval_accuracy': eval_accuracy,\n",
    "              'global_step': global_step,\n",
    "              'loss': tr_loss/nb_tr_steps}\n",
    "    \n",
    "    output_eval_file = os.path.join(args.output_dir, \"eval_results_ep_\"+str(eval_step)+\".txt\")\n",
    "    print(\"output_eval_file=\",output_eval_file)\n",
    "    with open(output_eval_file, \"w\") as writer:\n",
    "        logger.info(\"***** Eval results *****\")\n",
    "        for key in sorted(result.keys()):\n",
    "            logger.info(\"  %s = %s\", key, str(result[key]))\n",
    "            writer.write(\"%s = %s\\n\" % (key, str(result[key])))        \n",
    "def accuracy(out, labels):\n",
    "    outputs = np.argmax(out, axis=1)\n",
    "    return np.sum(outputs==labels)\n",
    "\n",
    "def frozen_model(bert_config, label_list, args, global_step, num_train_steps, device):\n",
    "    \"\"\" \n",
    "    \"\"\"\n",
    "    no_decay = ['bias', 'gamma', 'beta']\n",
    "    new_model = BertForSequenceClassification(bert_config, len(label_list))\n",
    "    new_optimizer_parameters = [\n",
    "                {'params': [p for n, p in new_model.named_parameters() if not any(nd in n for nd in no_decay)], 'weight_decay_rate': 0.01},\n",
    "                {'params': [p for n, p in new_model.named_parameters() if any(nd in n for nd in no_decay)], 'weight_decay_rate': 0.0}\n",
    "                ]\n",
    "    \n",
    "    new_optimizer = BERTAdam(new_optimizer_parameters,\n",
    "                            lr=args.learning_rate, \n",
    "                            warmup=args.warmup_proportion,\n",
    "                            t_total=num_train_steps,\n",
    "                            decay_factor=args.decay_factor)\n",
    "    torch.cuda.empty_cache()\n",
    "    new_model.load_state_dict(torch.load(args.output_dir + \"/checkpoint-{}.pth.tar\".format(int(global_step / args.grad_eval_step)))['state_dict'])\n",
    "    new_model.to(device)\n",
    "    model = new_model\n",
    "    optimizer = new_optimizer\n",
    "    return model, optimizer"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "05/25/2024 20:00:27 - INFO - __main__ -   device cuda n_gpu 1 distributed training False\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0\n",
      "guid= train-0\n",
      "text_a= Well, if you are looking for a great mind control movie, this is it. No movie has had so many gorgeous women under mind control, and naked. Marie Forsa, as the busty Helga, is under just about everytime she falls asleep and a few times when she isn't. One wishes they made more movies like this one.\n",
      "label= 1\n",
      "1000\n",
      "guid= train-1000\n",
      "text_a= Even thought I'm not the biggest of Cher fans, this movie was her crowning achievement. Granted, there were long term side-effects and risks of brain damage, memory loss (and) intellectual impairment, upon the screening such a film. A 1989 survey of Moonstruck fans by the UK Advocacy Network revealed that one-third of 300 Moonstruck fans surveyed believed Moonstruck had damaged them and an astounding 80% claimed it had irreparably destroyed their minds. Cher plays someone very un-Cher in this movie, a dowdy young widow named Loretta living in New York with her extended family. They're anti-American, pro-Italian and always at each other in someway. She has been going out with Johnny Camarary for a while, a nice mamma's boy man, and he asks her to marry him. She says yes. I loved her mom's questions: \"Do you love him Loretta?\", \"No.\", \"Good. If you love him he'll drive you crazy because they know they can. But you like him then?\", \"Oh yeah, he's a sweet man Ma\". When Johnny goes off to Sicily to care for his dying mother, he asks that Loretta make contact with his brother who he's been estranged from for years. This victory for human rights carries even greater significance, as Sicily was the birthplace of electroshock treatment. In 1938, Italian psychiatrist Ugo Cerletti, saw slaughterhouse workers using electric shock devices to cause epileptic fits in pigs, easing the job of slitting their throats. Cerletti was inspired, and began experimenting with electroshock on humans, developing the first Electroshock machine. Broken bones and fractured vertebrae that resulted from the convulsions appeared to be of little concern. This was,in so many ways, an anti-American movie. It's about love, to be sure, but it's also about infidelity, secrets, lonely people, and strange behavior brought on by American policies. The characters, from the frumpy BoBo at the favorite restaurant, the aunt and uncle, her parents and their problems, the ancient grandfather and his dogs are all well developed and intrinsic characters. It's somewhat of a chick flick, as it's how Loretta stops being a dowdy stuffed shirt and awakens the flower of the inner vamp. It's a Cinderella story in many ways, and that is every little girl's dream to emerge from the ugly duckling into a beautiful swan... Assuming free and fully informed Consent, it is well to reaffirm the individual's right to pursue happiness through brain damage if he or she so chooses. But we might ask ourselves whether we, as fans of cinema, though in no way sworn to any Hippocratic Oath, should be offering it.\n",
      "label= 0\n",
      "2000\n",
      "guid= train-2000\n",
      "text_a= Elizabeth Rohm was the weakest actress of all the Law and Order ADA's and her acting is even worse here. Her attempts at a Texas accent are amateurish and unrealistic. Nor can she adequately summon the intense emotions needed to play the mother of a kidnapped child; at times while her daughter is missing she manages to sound only vaguely annoyed, as if she can't remember where she left her keys. This is an important true story, so it's too bad that the awful acting of the lead actress distracts so much from the message. The rest of the cast is talented enough, but they just can't overcome Rohm's tendency to simply lay on a particularly thick imitation of a Southern drawl whenever actual acting is required.\n",
      "label= 0\n",
      "3000\n",
      "guid= train-3000\n",
      "text_a= What can you say about this movie? It was not terrible, but it was not good! Two days earlier I had watched Lillies and that was one of the best Gay films I have ever seen. So this was not the best time to watch a mediocre Gay flick. The story was silly and the acting was OK. It was not bad enough to turn off, but it had some bad moments and some terrible stereotyping. It was not very well cast either. Would I recommend this movie? No you would be wasting your time and money. I don't understand why movies like these are made and who is funding them. Spend your time Watching Noah's Arc on Logo instead. I think this is where this movie was trying to go but never got there.\n",
      "label= 0\n",
      "4000\n",
      "guid= train-4000\n",
      "text_a= I like animated shows. I enjoy the Nick fare pretty much, including Hey, Arnold. But moving a TV show to the Big Screen isn't easy and this just didn't feel big enough. It was more like a long episode of the show, and it just didn't move along that well. Judging by the behavior of the kids we had with us, it didn't score that well with them either.\n",
      "label= 0\n",
      "5000\n",
      "guid= train-5000\n",
      "text_a= DD films were damn corny, damn stupid and had a plot which seemed wafer thin but those days they was a plot at least This film isn't just a comedy but a mix of melodrama, romance everything Every drama scene is blown out of proportion The comedy is funny but corny too Yet the film keeps you entertained, those days Govinda films were loud, crass yet they had some funny moments people enjoyed David Dhawan does a okay job Music is okay Govinda acts well in comedy and drama Karisma is decent in parts and annoys in parts Kader is as usual Gulshan, Prem Chopra are typecast Shakti is hilarious\n",
      "label= 0\n",
      "6000\n",
      "guid= train-6000\n",
      "text_a= This was shown as part of the 59th Edinburgh International Festival, though for reasons best left to the powers that be. A lot seems to have been made of the fact that it's the first Thai language film, made with Thai actors & crew, but directed by a westerner. Needn't have bothered to be honest, as this film is dull, dull, dull. Why hint at something, why shroud an idea in mystery, why subtly invoke a feeling, when you can hammer the point home with terrible voice overs, obvious shots and over the top scenes> Nothing is left to the imagination as time and time again director Spurrier clumsily churns out endless clichés. No hinting, no guessing, it's all up on screen, no need to use our imaginations. Wonder when the 'scary' bit is coming? No you wont, 'cause the soundtrack will get more and more intimidating, rising to a crescendo of ominously. Hell, I'm making up words to describe how bad this is. Wonder whether the conjured demon is real or imaginary? Why tax yourself - it's really is a snake, and yes it's really is biting his crotch, and there's blood splattering everywhere. it's a strange, uneasy film for several reasons. It's supposed to be a horror film, but it's not scary - the jolts are signposted & obvious. It might be a scathing attack on the seedier side of Thailand, yet the director has a sleazy, lubricious style when it comes to showing barely pubescent teens. Maybe it was casting himself as the virginity-taking westerner that planted the seeds of doubt in my head. Or maybe the whole thing was just pants. Uninspired, insipid, repetitive, hackneyed - all candidates for best description, but dull seems most appropriate and honest. It's all been seen before, probably better, often with more thought, rarely with less imagination or flare. Sorry. Thumbs down on every count. Truly dire.\n",
      "label= 0\n",
      "7000\n",
      "guid= train-7000\n",
      "text_a= I like the good and evil battle. I liked Eddie in this movie better than any movie he has ever done. He wasn't The smart, cocky, know it all he usually plays. He shows heart and a more humble humor. The fact that it shows there are stranger things in Heaven and on earth than we can think of gives me hope.\n",
      "label= 1\n",
      "8000\n",
      "guid= train-8000\n",
      "text_a= The script for this TV soap opera is so bad that even A. Hopkins at some point had to play like an undergrad drama-student so as to bring some life in his script-dead character. I do not know whether this was the purpose of the director, but Hopkins' Ciano reeked nothing but vanity, fear and lack of self-esteem. The real Ciano possibly was all that but then, why make a movie about him? Mussolini was a bit more convincing, and his long way down was as if closer to the truth. Edda Mussolini was plain ridiculous (not because of Sarandon, but because of the impotent script), while she had to be the central character of this alleged familial drama. Watch it only if you enjoy Venezuelan soap opera.\n",
      "label= 0\n",
      "9000\n",
      "guid= train-9000\n",
      "text_a= What more could anyone want? He's a history lesson, foreign language tutor, NRA representative and ambassador to Burundi dressed in a nice silk frock and heels. I laughed so hard I left a puddle. His woes about puberty, transvestism, public school, and done in several languages made the absolute finest stand-up routine I have ever seen. I think about it now, years later when I see cake (tea and cake or death) and hear something translated into French (the mouse is under the table, the cat is on the chair and the monkey is on the branch. I like his versions of what Jerry Dorsey could have been named before he settled on Englebert Humperdinck. I really hope to see a lot more from this wonderful guy. He has a lot to teach us, and a wonderful way of telling it. Thanks for your time.\n",
      "label= 1\n",
      "10000\n",
      "guid= train-10000\n",
      "text_a= I'm surprised about the many female voters who even give this film better marks. My thought about this film was that the target audience is adult and male. Whipped and tortured women, merciless revenge and a high body count are typical ingredients, introduced into film history by the spaghetti subgenre. The opening and the hand-smashing are DJANGO rip-offs. THE SHOOTER however lacks the style of e.g. DJANGO. Score, acting and cinematography are mediocre at best but if you look for the above mentioned ingredients you are in the right place here. And the actors don't have an Italian accent. 4 / 10.\n",
      "label= 0\n",
      "11000\n",
      "guid= train-11000\n",
      "text_a= I first saw Jake Gyllenhaal in Jarhead (2005) a little while back and, since then, I've been watching every one of his movies that arrives on my radar screen. Like Clive Owen, he has an intensity (and he even resembles Owen somewhat) that just oozes from the screen. I feel sure that, if he lands some meaty roles, he'll crack an Oscar one day... That's not to denigrate this film at all. It's a fine story, with very believable people (well, it's based upon the author's early shenanigans with rocketry), a great cast  Chris Cooper is always good, and Laura Dern is always on my watch list  with the appropriate mix of humor, pathos, excitement...and the great sound track with so many rock n roll oldies to get the feet tapping. But, this film had a very special significance for me: in 1957, I was the same age as Homer Hickham; like him, I looked up at the night stars to watch Sputnik as it scudded across the blackness; like Homer also, I experimented with rocketry in my backyard and used even the exact same chemicals for fuel; and like Homer, I also had most of my attempts end in explosive disaster! What fun it was... I didn't achieve his great (metaphorical and physical) heights though. But, that's what you find out when you see this movie. Sure, it's a basic family movie, but that's a dying breed these days, it seems. Take the time to see it, with the kids: you'll all have a lot of good laughs.\n",
      "label= 1\n",
      "12000\n",
      "guid= train-12000\n",
      "text_a= I watched this film on the Hallmark Channel recently. In my opinion, the film started out decent enough, but eventually got sour. The Story: A U.S. soldier in Afghanistan receives one of the Christmas cards that a woman back in the U.S. has sent out to troops for Christmas. He becomes so inspired by the Christmas card that he feels it has given him a ray of hope and happiness in his life, a motivation to continue on. When he is given leave, he goes to the very town that the woman lives in. He comes across and spunky young woman and eventually finds out that she is the one that sent the Christmas card. He meets the family and after saving her dad (a Vietnam War Veteran) from getting hit by a car whilst crossing the street, the family decides to take him in for a while. We learn that he doesn't have a family back home. The soldier agrees to help out the family during the Christmas season by working with them at their logging company. The family comes to love him and vice-versa. The soldier also becomes in love with the family's daughter - the woman that sent the Christmas card. There's only one problem. She already has a serious relationship with a man that she's in love with. It's also a long distance relationship. The boyfriend is very different from the soldier. He prefers wine and trips to France over hard labor and the great outdoors. The woman prefers the latter. Throughout the rest of the film, there is a \"love triangle\" as the woman's boyfriend is back in town visiting her for the season. Everyone in the family seems to want the daughter to be with the soldier and the mom basically wants what's best for her daughter. **(SPOILER ALERT)** The woman comes to like the soldier more and more, as they have a lot in common. They happen to spend a lot of time together. She gradually becomes attracted to his persona. Then one day he kisses her and and she kisses him back. Now she's so confused! She's in love with her boyfriend and wants to marry him someday, but she's become so fond of the soldier. Soon enough, the boyfriend (who doesn't know about the kiss, but is very protective of the woman and thinks the soldier has been trying to move in on her) decides to propose to the woman and she accepts. His plans for their marriage, however, are not plans that she wants. She wants to be at home, have kids and be close to her family. He wants to travel the world, go to exotic places and if they have kids he wants to take them along. This doesn't sound too thrilling for the woman. However, the soldier feels bad about what he's done and doesn't want to make things worse. So, he decides to leave town, despite the dad's urging him to stay and be with his daughter. It all comes to a head when, at the Christmas Eve service with Church, we find out that the soldier hasn't left yet. The boyfriend and the woman speak to each other outside and he, broken-hearted, basically decides to let her go and breaks up with her. The film ends with the woman and the soldier getting together. Things I like: I like the soldier throughout much of the film. He's basically nice and polite, with a strong sense of doing good for others; there are moments in which the family is going with Church and participating in charitable causes; the family hold hands and pray during an evening meal at the table; the dad and mom have a long-lasting and evidently healthy marriage; the soldier tries to do the right thing in a situation after he did something very wrong  Things I don't like: There is a very typical \"love triangle\" story in this film; the soldier turns sour as he slowly and subtly moves in on the woman and then eventually kisses her when she already has a boyfriend; most of the family practically applauds this behavior - the family goes with Church and participates in services, yet this is how they act; there is some bad language  Conclusion: I became disappointed as the film ran on, and it got worse and worse. I ended up fast forwarding the very end of the film because I had enough. So, the film started out decent with some few gem moments, but ended up ruining itself. Therefore, I don't recommend this film to anyone.\n",
      "label= 0\n",
      "13000\n",
      "guid= train-13000\n",
      "text_a= The pioneering Technicolor Cinematography (Winner of Special Technical Achievement Oscar) is indeed enchanting. Add an endless variety of glamorous costumes and a romantic cinema dream team like Marlene Dietrich and Charles Boyer, and you've got a rather pleasant \"picture\". Unfortunately the contrived plot as well as the over-blown acting leave much to be desired. Still, there have not been any more breathtaking Technicolor films before this one (1936), and very few since then, that can top this breathtaking visual experience of stunning colors. Cinema fans who have enjoyed the glorious color cinematography in \"Robin Hood\" (1938), \"Jesse James\" (1939) and \"Gone With The Wind\" (1939), will not be disappointed in the fantastic work done here. \"The Garden Of Allah\" will always be synonymous with brilliant color cinematography.\n",
      "label= 1\n",
      "14000\n",
      "guid= train-14000\n",
      "text_a= I sat down to watch a documentary about Puerto Rico, and I ended up watching one about Nuyoricans. When I go to Puerto Rico, I fail to see the 50% that live in poverty. When I do see struggling people, they are usually Haitians, Dominicans, or Cubans that have recently arrived to the island. There is no such thing as spanglish... either you speak Spanish, or you don't.... and from what I heard... you don't. Pedro Albizo Campos IS NOT MLK to me. MLK was a great man. Campos is a great man to those that want independence which is 1%. To the rest he as loco as Osama Bin Laden. Puertoricans that want independence are a bunch of fools. If you want any proof to all of you dreamers of an independent Puerto Rico see Cuba, Haiti, Dominican Republic, Bahamas, all of South and Central America, and Mexico. Its worked wonders for them. This documentary is not about Puerto Rico, this documentary was about the Nuyoricans and their struggles. To the person that complaint that not enough of Africa was on the show... it was suppose to be about Puerto Rico... not Africa. Denzel will make one shortly just for you. In conclusion... to all those ignorant white people that think we need green cards to come to the US, and want to learn how the prime minister runs things, this is not a good documentary about Puertorican culture. Tell your kids to pay attention in Geography, and History class. ***Update*** Bocabonita... \"doc.\" was about Nuyoricans. She promoted it as if its how we all feel. Should have been titled... \"yo soy nuyorican... lunche...can't speak Spanish.\" PLEASE STOP USING PUERTO RICO, RICAN, BORICUA, OR ANYTHING ELSE ASSOCIATED WITH PR WITH THIS NUYORICAN HISTORICAL LESSON. God forbid they play this on the island.\n",
      "label= 0\n",
      "15000\n",
      "guid= train-15000\n",
      "text_a= Why oh why do people take good material and feel the need to change it some how? Having read the book on which this film is allegedly based a couple of years ago, I can say that there is little if anything from the original book. I went into this film with low expectations - i knew it would have a crappy telemovie feel to it - but it even failed to meet these. This is not a prequel - the only relationship it has with the original is the name. This is not the story of Carlito Brigante, it is the story of a totally different character who's been given the same name. They have just totally spat in the face of every Carlito's Way fan out there; adding insult to injury by casting Luis Guzman, who plays a crucial character in the original movie, in a different role. What's most disappointing is that now that this film has been made no other film will be made addressing the original, untouched material of the book Carlito's Way - something I really would have liked to have seen. I felt the same way about Chopper - they have four books as well as interviews worth of fantastic non-fictional material and could have made a brilliant biography of Australia's most feared underworld figure, instead they made a ho-hum film about a deranged but strangely pathetic small time crook (though Eric Bana's performance was spot-on). Now we will never get to see Carlito's real initial rise and fall. The three stars is because, looking at it purely as a stand-alone flick, it is not so appalling - there are some decent performances (Jay Hernandez is no Carlito but he could be good in other roles) and the story is not too bad. Even Puffy Combs suits his role. But they totally misunderstand the nature of the underworld at that time (I am a bit of a crime-non-fiction buff)- something which the original film and the books got right (having been written by then attorney and now judge Edwin Torres). The fact is though, it's not a stand alone - it's perhaps the most disappointing prequel ever filmed.\n",
      "label= 0\n",
      "16000\n",
      "guid= train-16000\n",
      "text_a= Even if one didn't realize that Sellers was in poor health at the time of filming and passed away before the film's release, THE FIENDISH PLOT OF DR. FU MANCHU would be painful viewing. It is supposedly a lampoon of Sax Rohmer's famous Oriental villain but it lacks any focus. The potential for satirical commentary on the anti-Oriental overtones of Rohmer's concept are ignored. Indeed, the movie employs racist insults. There are hardly any actual jokes or gags, just mostly actors behaving idiotically and spouting dreary lines. It is especially distressing to see Sid Caesar forced to spout curses and racial slurs for attempted laughs. Most of the other actors embarrass themselves as well. And then there's Peter Sellers. He plays the dual roles of the sinister Fu Manchu, who is trying to concoct a formula to regain his youth and his stalwart British foe Nayland Smith. Sellers isn't one hundred per cent bad; he conveys a quirky warmth as Smith when he discusses his fetishistic attachment to his lawn mower and he's oddly moving as Manchu when he expresses his love for English music hall entertainment. But most of the time, he plays both roles with a weary grimness, thus further sabotaging any comical possibilities. Sellers' routines where he revitalizes his fading strength with electric shocks are particularly excruciating; he seems too convincingly agonized to be funny.  A few genuinely witty lines, an apt slapstick bit by Burt Kwouk (Cato in the PINK PANTHER films) as one of Manchu's minions, and Helen Mirren's amusing musical numbers cannot salvage this mess. If anyone wants to understand why Peter Sellers is considered a comedic genius, they won't learn anything from THE FIENDISH PLOT OF DR. FU MANCHU.\n",
      "label= 0\n",
      "17000\n",
      "guid= train-17000\n",
      "text_a= When I heard that this movie was coming out the night before Halloween, I was very excited. When I found out that it was a book, written in 1978, I had to read it before seeing the movie. I'm sure the movie would have been much different to me if I had not read the book. The writers actually did a good job of staying true to the main plot of the book, with minor differences, naturally. I think the thing that disappointed me the most about the movie was Boyle playing the role of Col. I'm not a big fan of Boyle, and it seems that no matter what the mood during the movie, she's always trying to use her over-plumped lips, and darkly makeup-ed eyes to make herself seem super sexy. Indeed, I think that the movie held true to the genuine creepiness of the house. My favorite subplot was the Sheehan family (which is so weird b/c the son was killed in Iraq and in current events there is Casey Sheehan whose mother went on a huge anti-Iraq tirade). In the book, obviously the war was not Iraq, but rather, Vietnam, and when the house turns on that video of the son in the helicopter, I was truly creeped out. Overall, I was impressed with the movie, in that it followed the book very well.\n",
      "label= 1\n",
      "18000\n",
      "guid= train-18000\n",
      "text_a= Much to her adult children's chagrin & nearly immediately after Elizabeth's (Dame Judy Dench) husband's death, the widowed, attic tenor saxophone player becomes bent upon openly returning to her musical hobby. Now that George is dead, Elizabeth no longer has to practice playing sax in the attic. As she grows more pleased with playing in the open, Elizabeth takes a stroll along memory lane, remembering when she was a 15 year old member of a jazz swing band, \"The Blonde Bombshells\": supposedly, an all-girl WWII group of talented jazz swing musicians. One of the \"Blonde Bombshells'\" band members was a womanizing, cross-dressing drummer, Patrick (Ian Holm), with whom Elizabeth remained friends. Both Patrick & Elizabeth's 12-year-old grand-daughter, Joanna (Millie Findlay), press Elizabeth to round up the former band members & take up performing together again; this time as a bunch of sexagenarians. Among the band members she finds are the (still foxy!) bass playing, Madeleine (Leslie Caron); Dinah (Olympia Dukakis), a trumpet playing, alcoholic & out-spoken, money-grubbing divorcée & widow living off of wealth from her many (ex)marriages in a Craigievar Scottish castle; Gwen, (real life US star jazz singer, Clio Laine), having at the lead vocal; Annie, (June Whitfield), as the Salvation Army trombone player; Betty, (the late piano player, Joan Sims), who's located training the ivory keys in a Hastings pub. As Elizabeth, Patrick & Joanna scout the world for members of the 1940's band & try to convince them to resume performing together, Elizabeth is oft times beside herself as she learns more than she wants to know about their adult lives--including her own--while having a blast playing terrific music with the last of the living 'Blonde Bombshells'. Amusing, nostalgic, historical, sentimental, multi-generational entertainment that is seriously fun. The actors deliver wonderful performances. Regardless of their ages, they are still Bombshell entertainers who put on quite a show. (The DVD is now out & worth owning because of the bonus features & Dolby Digital sound). Surely as a fan of any of these terrific actors the VHS is a collector's item.\n",
      "label= 1\n",
      "19000\n",
      "guid= train-19000\n",
      "text_a= Before Tuscan Sky, I saw Diane Lane's tender performance in this otherwise lark of a movie. Campers are invited to the camp of their youth and experience it as adults. Each of those that return seem to be looking for something they lost, which makes it so realistic. Maybe you had to be a camper to really get it, but in the words of one character noticing all her clothes were wet \"this is so camp!\" From the practical jokes and fighting over boyfriends, to the scary lunch lady and the early morning bell ... it's amp. Once exciting activities now seem mundane. A terrific ensemble cast makes the best of one-two dimensional roles and makes them believable. Bill Paxton, Diane, Elizabeth, Mrs. Brad Paisley (probably when he first fell for her!!) The beautiful scenery, bright colors, comical music (including variations of Hello Muddah)and a comic acting turn by noted director Sam Raimi makes this a movie you can pull out again and again like looking up an old friend.\n",
      "label= 1\n",
      "20000\n",
      "guid= train-20000\n",
      "text_a= the movie opens with a beautiful lady in a tattered white gown running through a stereotypical eastern european town. we know she's being followed by something, because she keeps looking behind her. and soon we see she's being chased by a mysterious man in a black trenchcoat. then we realize that the man is actually the vampire hunter and he is after her. but look is that her reflection in the store window??? no its just her identical twin vampire! but unfortunately they both get it. after this brilliant and amazingly fun throwback to the old hammer films of the 60's and 70's (in the credits the twins are listed as the twins of evil, which of course is the name of the final instalment in hammer's karnstein trilogy), the plot pretty much dies. What little plot there is involves dracula (who conveniently changes his appearance each time he is reborn, so the producer doesn't have to rehire the same dracula) coming to a morgue, the med students realizing he's undead and thinking....wow what an opportunity, maybe i'll just disregard all those movies that say that drinking vampire blood turns you into a vampire and use the vampire's blood to find a cure for our jerk friend's ailment. obviously this is a mistake and everyone becomes a vampire. A new concept but pulled off excruciatingly badly. The movie keeps setting up wonderful situations and refuses to do anything with them. For example the med students attempt to bring drac back to life by placing him in a bathtub filled with blood in a secluded run down country mansion. The house itself is scary enough to be the center of the film, but do we stay there? no because they decide to take the vampire to an abandoned swimming pool. sigh. This movie has a real problem with \"homages\" as i mentioned before the opening scene is straight out of hammer, and this house scene would have been perfect for a hammer-like movie, but the movie rapidly switches gears and changes to a medical horror. The other problem is that they introduce so many characters it is almost impossible to feel sorry for any of them. There are the med students and their wheelchair bound professor-type \"friend\" the med students are all: arrogant, boring, money hungry, and stupid. how they made it to med school at all amazes me, unless the med school had to meet its muscle bound hunk/big breast quota. and then there is the vampire hunter who remains mysterious through the movie. hey i can respect that but it would be nice if they didn't set it up like the movie would be about him. then you have random priests, cops, and science types. so many people are introduced and then quickly forgotten about until they need that person to either save the day or jump out for a cheap scare that it becomes quickly tedious. Basically this is a lazy movie. no real scares, just a few predictable jump scares. The set up for these is so elaborate it is hilarious. for examp le the bathtub full of blood. it is so obvious that drac is going to pop out of the murky blood. and yet we have to wait far too long to get to the inevitable jump scare. after this he kills one of the dumber and larger breasted med students. we all know she's going to become one of the undead. but what do the others do? bury her in a shallow grave near the house. sigh, so you know who will jump out at you when the cops show up at the house.......... Oh well. Maybe someone will get the hint that it is impossible to make a scary vampire movie and just go for atmospheric, and then we will end up with an entire movie that is as good as the opening scene. \n",
      "label= 0\n",
      "21000\n",
      "guid= train-21000\n",
      "text_a= A few weeks ago, I read the classic George Orwell novel, 1984. I was fascinated with it and thought it was one of the best books I've read recently. So when I rented the DVD, I was intrigued to see how this adaptation measured up. Unfortunately, the movie didn't even come close to creating the ambiance or developing the characters that Orwell so masterfully did in his book. The director seems to think that everyone watching the movie has read the book, because he makes no attempt to demonstrate WHY the characters act and feel the way they do. John Hurt, the main actor, is droll the entire way through, and hardly does any acting until the end. We never really find out what he does for a living, or why his love affair is forbidden, or what the political climate is and why the main character desires rebellion. This book cannot be done justice in movie form without proper narration and explanation of the political system oppressing the characters, and the fact that those are missing is the greatest shortcoming of this film. Besides that, John Hurt was a terrible casting choice, looking about 15 years older than the 39 year old Winston he was supposed to be portraying. On a more positive note, however, the rest of the cast was well chosen. It's just too bad they were put in such a horribly adapted film with the wrong lead actor. -Brian O.\n",
      "label= 0\n",
      "22000\n",
      "guid= train-22000\n",
      "text_a= I sat glued to the screen, riveted, yawning, yet keeping an attentive eye. I waited for the next awful special effect, or the next ridiculously clichéd plot item to show up full force, so I could learn how not to make a movie. It seems when they set out to make this movie, the crew watched every single other action/science-fiction/shoot-em-up/good vs. evil movie ever made, and saw cool things and said: \"Hey, we can do that.\" For example, the only car parked within a mile on what seems like a one way road with a shoulder not meant for parking, is the one car the protagonist, an attractive brunette born of bile, is thrown on to. The car blows to pieces before she even lands on it. The special effects were quite obviously my biggest beef with this movie. But what really put it in my bad books was the implausibility, and lack of reason for so many elements! For example, the antagonist, a flying demon with the ability to inflict harm in bizarre ways, happens upon a lone army truck transporting an important VIP. Nameless security guys with guns get out of the truck, you know they are already dead. Then the guy protecting the VIP says \"Under no circumstances do you leave this truck, do you understand me?\" He gets out to find the beast that killed his 3 buddies, he gets whacked in an almost comically cliché fashion. Then for no apparent reason, defying logic, convention, and common sense, the dumb ass VIP GETS OUT OF THE TRUCK!!! A lot of what happened along the course of the movie didn't make sense. Transparent acting distanced me from the movie, as well as bad camera-work, and things that just make you go: \"Wow, that's incredibly cheesy.\" Shiri Appleby saved the movie from a 1, because she gave the movie the one element that always makes viewers enjoy the experience, sex appeal.\n",
      "label= 0\n",
      "23000\n",
      "guid= train-23000\n",
      "text_a= I watched this movie last night and was a bit disappointed. A lot of the \"time facts\" were off. At the beginning of the movie, the grandfather made a comment to this grandson and his friends about how they felt when 9-11 hit. This movie was supposed to be taking place in 1994. Also, one of the grandsons friends was wearing an Eagles Donovan McNabb jersey. He hadn't even been drafted as of yet. The story line was good but the facts and actuality of the time frame was so far off base that it made the movie seem cheesy. My boyfriend is an avid reader of WWII books and memorabilia. I rented this movie hoping that it would be good. The acting was so-so. The dog was cute. But the way that this movie was carried out made me glad that I only paid 4 dollars to rent it as opposed to the 50 it would have taken me to watch it in a theater. I hope that who ever reads this understands that I mean no discontent to those who fought the war but the facts and time frame should have been a little more closely monitored.\n",
      "label= 0\n",
      "24000\n",
      "guid= train-24000\n",
      "text_a= Bela Lugosi plays Dr. Lorenz who loves his wife so much that he will do anything to keep her young. This film starts off with a wedding as the bride is about to take her vows she suddenly collapses. She is pronounced dead and taken away by undertakers. Trouble is that these are not real undertakers but body snatchers. A wave of bride deaths at the altar and their body disappearing confounds the police. Enter reporter Patricia Hunter to solve the case. She does track down Dr. Lorenz but he also decides to use her youth to keep his wife young also.\n",
      "label= 0\n",
      "Seeded everything\n",
      "0\n",
      "guid= dev-0\n",
      "text_a= The Mascot is Ladislaw Starewicz's masterpiece. It tells the tale of a stuffed toy dog who searches for an orange after he overhears the mother tell her daughter she hasn't any money. The dog gets picked with other toys to be sold off. In the truck, after the others jump off while the vehicle runs, the dog stays and waits to be picked up from store before setting off on his own. He manages to get orange after biting woman's leg as she was holding and selling the fruit. As he runs, he encounters the devil and accepts his offer to stop at nightclub. There, he meets the other toys who jumped off truck. The cat who was next to him in vehicle is especially determined to get dog's orange. I'll just stop here to mention that other wonderfully bizarre things happen in nightclub that you'll have to see for yourself. Suffice it to say, if you love Starewicz and is interested in all animation from the past, I most definitely recommend you seek this one out!\n",
      "label= 1\n",
      "1000\n",
      "guid= dev-1000\n",
      "text_a= The banter and humorous rescue scene help to make this one of my favorites of the 14-movie series. Wonderful acting, great cast. And this movie contains one of the few oft-noted facts about Sean Bean's career. The part where he and Alice Krige fall off the horse into the water was not scripted but was left in since they both went right on acting after it happened. This is a good follow-up to the intense ending of Sharpe's Enemy.\n",
      "label= 1\n",
      "2000\n",
      "guid= dev-2000\n",
      "text_a= I don't dislike Cary Grant but I've found his performances annoying in enough films to notice; this, Arsenic & Old Lace and Bringing up Baby. I don't dislike him in North by Northwest but I really find that movie unbearably silly. On top of that I find the endless raving about Grant's class tiresome. I don't have a clue what his class does for the viewers who herald it. It doesn't do a thing for me. In the behind-the-scenes feature included with this DVD Patrcia Hitchcock says that Grant was her fathers favorite leading man; I think he was wrong. Jimmy Stewart was a better leading man in a string of better Hitchcock movies. With it's ruined ending this is really half a movie and doesn't bear discussion, and can't support the high ratings it's getting. Even if the movie had it's ending intact there's not much to it. Fontaine is a completely unsympathetic sucker. She has to remain numb, inactive, and unwilling to contact anyone but Johnny for the whole movie, in either ending, for his ploys to work. That's not much to work with. Cary Grant begins every line with \"Monkeyface...\" until I wanted to strangle him. He says it about sixty times. It's positively grating. Hitch's technique here is shockingly shallow. An endless succession of rooms/sets have a phony skylight projected on the rear wall as a spiderweb effect. And a light-bulb in a glass of milk may make fans excited, but it can't save a movie this poorly made. Peter Bogdanovich should retire if he does one more Hitchcock/Cary Grant imitation on a DVD. I think that's his whole career now. As soon as I saw him, I thought, oh crap, here comes an imitation that only he's impressed with. Instead there were two! oh joy!\n",
      "label= 0\n",
      "3000\n",
      "guid= dev-3000\n",
      "text_a= I would label this show as horrendous if it weren't for the fact that it's on the same network as Arrested Development. Because it is on FOX and getting renewed while AD got cancelled.  It is absolutely beyond words how atrocious this show actually is. But let me try and describe it. Take an extremely low rate Archie Bunker and have him spout out humor that would have been out of date if it were on Married with Children. Then take great plot lines from AD (son has an ugly, boring girlfriend) and dumb them down so the idiots who watch sitcoms can understand them.  If you watch this, I will have completely lost respect for you, as should your family. However, if you are a fan, you should love FOX's new comedy 'Til Death. Looks like real funny, cutting-edge stuff. I mean, married couples not getting along ... brilliant.\n",
      "label= 0\n",
      "4000\n",
      "guid= dev-4000\n",
      "text_a= I'm a big fan of surrealist art, but this film by Bunuel (with some ideas from Dali) left me cold. Bunuel had a life-long grudge against the Catholic church and delighted in trying to offend Catholics in fairly silly ways. This is one of the silliest; almost like what you'd expect from a smart-aleck 18-year-old in film class. The last few minutes of the movie, which have nothing to do with anything else, are a final nose-thumbing at religion. If you read the \"scholars\" regarding this slow-paced, occasionally amusing film, it's all about how the church and society are guilty of sexual repression. If that is indeed the point, then Bunuel expresses it in the most roundabout fashion possible. The central male character is a nasty brute who loves kicking dogs and knocking blind men down in the street, and who mentally turns billboard ads into strange sexual fantasies. Is this behavior the church's fault (for interrupting his lovemaking), or is he just a jerk? I vote for the latter. I think Bunuel must have had a lot of personal hangups and chose the Catholics as the ones to blame. There are a few moments where you might cry, \"Aha! surrealism!\": a cow in a bed, a giraffe falling out a window (a poor model), a man shredding a feather pillow, a woman flushing a toilet while we watch pictures of seething lava (or a mud pit...hard to tell in B/W). The rest is forgettable self-indulgence. Unfortunately, Bunuel was still chasing the same bogey-men through the rest of his career (Viridiana, Discreet Charm...). If you're interested in seeing surrealism on the screen, check out Jean Cocteau's early work.\n",
      "label= 0\n",
      "5000\n",
      "guid= dev-5000\n",
      "text_a= This film was terrible. I thought it would be OK but it just got worse and worse. From the starting scenes it seems to be heading in the direction of another safe predictable rom-com, but the moment he arrives at the house it just disintegrates. None of the characters have any depth and the only person who was anywhere near believable was Tom, although the way he became so easily distracted just annoyed me after a while. The dialogue is ridiculous and the structure of the film almost completely non-existent. In an insulting attempt at comedy the writer/director introduces a new character or event in practically every scene, none of which are realistic, making it very confusing to keep track of what is going on. The plot is barely an excuse for a movie : guy likes girl, house sits fathers home to get to know girl, destroys house, gets girl. A complete waste of time.\n",
      "label= 0\n",
      "6000\n",
      "guid= dev-6000\n",
      "text_a= This movie is so over-the-top as to be a borderline comedy. Laws of physics are broken. Things explode for no good reason. Great movie to sit down with a six-pack and enjoy. Do not - I repeat DO NOT see this movie sober. You will die horrible death!!!\n",
      "label= 0\n",
      "7000\n",
      "guid= dev-7000\n",
      "text_a= Combine good casting, bad writing, good orchestral scoring, bad dialogue, and good story idea with lots of potential but is never realized then you have Slipstream.  Just bought the movie for a buck, it is worth it, but not much more.  Good to see Mark Hamill act again.  There should be a decent sequel made to remedy the damage from the original. Or at least give it the proper attention it should have received in the first place.  Berstein's score gave demanded your attention from the opening credits, however, the long shots of slipstream planes and the even longer revealing of interesting plot points mutes his attention getting score.  It is really easy to dog a movie like this, after all it is by the producer of STARWARS and the director of TRON and a tremendous cast but it is what it is. And that ain't much. Favorite Line- \"We're going to make it, ha-ha!...(BOOM!)\"\n",
      "label= 0\n",
      "8000\n",
      "guid= dev-8000\n",
      "text_a= James Bridges' \"The China Syndrome\" is a first rate thriller; a model for those who want to make a genuinely terrifying thriller but don't know how. Most thrillers end with the standard shoot-em-up and chase that ends with the villain getting what he/she deserves. But Bridges understands that such a standard finale isn't the case in some scenarios. \"The China Syndrome\" is thrilling in a way no one would expect. It has the type of ending that's so unexpected, but yet so logical. The film stars three (then) current Oscar winners: Jack Lemmon (Best Actor 1973 for \"Save the Tiger\"), Jane Fonda (Best Actress 1971 and 1978 for \"Klute\" and \"Coming Home\" respectively) and Michael Douglas (Best Picture 1975 for producing \"One Flew Over The Cuckoo's Nest\". This isn't your standard \"spot the star\" flick that became so popular in the 70s. The acting is so solid and strong that we forget who's playing the roles and believe that the characters are real people. That's a testament to the collective ablities of all three actors. They may be stars, but they're actors first. There is no music score in the film. Asides from a few background songs from jukeboxes or TV shows, there is nothing. Bridges doesn't allow any directorial style to come through on screen. He is simply presenting the material straightforward. With a strong dramatic story like this, we don't need a score to distract us. It kind of strikes me as a predecessor to the Dogma 95 filmseries started by Lars Von Trier in 1997. It focuses on the characters and story rather than style. The story is about a nuclear power plant located in Southern California. **SPOILER** An accident appears to have happened and basically the film is about Lemmon,Fonda and Douglas try to expose the truth. In a way this is not a spoiler, since the trailer, TV spots, video boxes and reviews all give this away. But from this seemingly simple premise, a surprisingly complex morality play springs and the suspense comes from human nature and the actions of people rather than a villain framing another guy (although this is an element in the film). That's what makes \"The China Syndrome\" so good. **** out of 4 stars\n",
      "label= 1\n",
      "9000\n",
      "guid= dev-9000\n",
      "text_a= 2/3 of this movie is recycled footage of the previous movies, a fact that's sadly obvious even to someone like myself who hasn't seen the original movies. And somehow it feels like a rip-off even though I haven't seen the stuff before. It's like that episode of every TV show where the characters sit around a photo album or something and you just see recycled footage of other episodes. I've seen some producers do extended montages of recycled footage, but never anything beyond 5 minutes or so. This movie is mostly stuff that had already been seen by audiences, so you could mount a case that it's one of the biggest rip-offs ever foisted on the motion picture public. I got to see it in the theater, in a 16mm print, which is good enough I suppose considering how rare this kind of material must be on film these days. I give the movie some credit for semi-convincing Gothic atmosphere and for unintentional humor, but that's about it. The Aztec mummy monster looks good, even has some mobility in his face which is better than most movie monsters of the period. But the robot is pitiful, although it's interesting that they made the human face totally visible. It's a \"robot human\" or something of the sort as they somewhat explain in the movie. I think that's considered an android. So technically in hard sci-fi terms this movie should be called \"Android vs. the Aztec Mummy\", but I doubt anyone was too worried about technicalities here anymore than they were worried about quality. In fact the movie is so sloppily put together that it makes television look good. Even the dubbing from Mexican into English is lazy and weak -- for example at one point the hero says \"I might as well begin at the beginning....\" what the heck kind of translation is that? Couldn't they at least have him say \"start at the beginning\" so that it doesn't sound repetitive? A high school newspaper editor could have fixed the screenplay of this movie. It's the epitome of utilitarian film-making, just absolutely nothing is in this movie that doesn't need to be there for the basic commercial purpose of the film. They put no more effort into making it than they had to, and considering the extensive recycled footage I would doubt that they actually spent more than a week making this movie. I will now cease posting about it on the principle that I don't want to expend more energy in the process of commenting than the creators of the movie actually expended while making it.\n",
      "label= 0\n",
      "10000\n",
      "guid= dev-10000\n",
      "text_a= *** Contains spoilers *** A lovely film this, starring Brad Renfro and the ever wonderful Joseph Mazzello. I like Joseph Mazzello, out of all his films I've seen to date I've loved every single one of them for many different reasons and The Cure is no different. Brad Renfro does very well in this movie as well. The Cure is a drama/coming of age movie from the viewpoint of an ill child and his friend. The basic idea is: Dexter (Joseph Mazzello) has AIDS. He ends up befriending the kid next door (Brad Renfro) but Erik's mum is very narrow-minded, ill-informed and somewhat \"thick\" when it comes to Dexter's illness. She thinks AIDS is contagious like the Common Cold so doesn't want her son going anywhere near Dexter. After many attempts at making their own cure with no success, the boys go on their way to New Orleans to find the cure after reading a pamphlet about it. After getting their kicks from Playboy magazine, Dexter's health goes south shortly afterwards and as his health detoriates, there's still enough life in the boy alongside Erik for two pranks of pretending to stop breathing. Unfortunately, poor Dexter does indeed die from his illness, leaving poor Erik behind to wonder why he couldn't find the cure. Throughout the movie he ends up bonding more with Dexter's mother than his own. It is a very heartwarming movie to watch and is not absolutely perfect (movies rarely are) but you won't care less about that as you get involved in the film more. A must for Joseph Mazzello fans, one of his best performances ever. Very well recommended must-see movie - if you can find a copy :)\n",
      "label= 1\n",
      "11000\n",
      "guid= dev-11000\n",
      "text_a= Thorn-BMI is out of business, before they stopped making films they made a chiller of a movie. Using E.S.P. and telekinesis as the basis of the daughter whose father mastered a terrible power. Only in the death of her father did Olivia find that her father dubbed 'Raymar' from Raymarkovitch had really murdered 6 girls and was planning two more by using the technique of Psyhic Vampirism. Our picture starts with 6 coroner wagons pulling in and music to match the grusome discovery of the 6 girls. Dead all with their eyes wide open in a closet. In the walls were all kinds of objects, the coroners men were pulling up an old man, when blue lightning hit the ceiling which caused a circular hole to form only made the film more bizarre! If you like extremely chilling scenes this for you. Unless you can see dead bodies from years ago in each level of decay, don't view it without a friendly companion. Like \"The Changeling\" it has some heart stopping horror in it. I gave this a rating of 7 it's in color, actress Meg Tilly debuted in this film if you can find it see it.\n",
      "label= 1\n",
      "12000\n",
      "guid= dev-12000\n",
      "text_a= This is a very amazing movie! The characters seemed so realistic to me, it was hard to believe they weren't real people. Being from the South, I thought Judith Ivey's character seemed especially real, and as everyone else has mentioned, she does an outstanding acting job. The characters are not beautiful and look nothing like the average Hollywood stars - their imperfect bodies and personalities seem so much more natural and real.  One reviewer mentioned that the main character, Alice, had no good reason to run away from home, which is true - she didn't have any moral or upstanding reason to run away, such as escaping child abuse, etc. I thought that she was just fed up with dead-end jobs in a working class life and wanted to flee down to Florida where her friend lived the appealing and privileged life of a college student in Miami. The actress shows Alice's confusion, uncertainty, and questioning turn into decisiveness and willingness to take control of her life with impressive naturalness. The film also shows how Alice is trapped in situations with seemingly no options, causing her to panic, take action, and reach out for help.  At first, the grainy filming style put me off and made me think that it was a very low budget or homemade movie, but in actuality it is very well done. The home movie quality really makes you feel like you are there with the characters, a part of their RV trip across the country. This is definitely a film worth seeing, although I don't quite understand all the descriptions of it as a heart-warming coming of age tale. It is rather vulgar and disturbing at times, even if it is not completely sad in the end.\n",
      "label= 1\n",
      "13000\n",
      "guid= dev-13000\n",
      "text_a= In a farcical key, Gaudí Afternoons can be taken as a mediocre exercise. Marcia Gay Harden and Judy Davis pivoted a good cast (Juliette Lewis' new-age freaky character has been incredibly taken from reality, I know an American young lady who squawks like her!!) but GA does not show much beyond its overtoned plot. Even though movie-making is all about make believe, there were certain noticeable screenplay inconsistencies. Two samples: you pay 14 euro to enter the chapel where Cassandra and Frankie met, NEVER at 7 am, and you cannot leave a terrace without paying the bill (they'll charge you on the spot if they don't know you) or get off a taxi THAT quickly (you Americans always tip cabbies even though they don't expect to, but the sequences portrayed in the movie were ridiculous). Don't believe me, reader: come over and see for yourself. If you've never been here before you might not care about all this, but good movies should be believable disregarding of your origin. Nobody knows about GA here, and I will make sure that does not change in the future.\n",
      "label= 0\n",
      "14000\n",
      "guid= dev-14000\n",
      "text_a= Not for everyone, but I really like it. Nice ensemble cast, with nice contributions from better known players (like Stockard Channing) and strong eye candy (from Sheila Kelley). What really works is the bond between the three brothers! Try it, you'll smile a little.\n",
      "label= 1\n",
      "15000\n",
      "guid= dev-15000\n",
      "text_a= You'd hardly know that a year later MGM put Norma Shearer in THE DIVORCEE which glows with MGM technical know how. How far they came in one year. CHENEY is a very stagey early talkie. The camera hardly moves. Shearer is her usual charming self and Rathbone does well in a romantic leading role. They are all very careful to speak clearly and slowly into the microphone source which does mitigate against a naturally flowing dramatic scene, but the play is a sturdy and fun warhorse so one can enjoy oneself if one's expectations are not too high. Oh, by the way, the plot involves a ring of upper class jewel thieves who infiltrate themselves into society to prey on their victims. There are some clever twists in the script and true love conquers all. An Oscar nom for Best Screenplay Adaptation.\n",
      "label= 0\n",
      "16000\n",
      "guid= dev-16000\n",
      "text_a= Many movies try to take universal themes and make a comedy; but few will rise to the occasion like \"Checking Out.\" The movie is brilliant. The dialogue is well written and true to form. The acting is absolutely prima. Peter Falk has given a truly great performance - as an actor; as an actor. He is able to carry the cast to greatness. Another great performance is given by Laura San Giacomo. She is such an intriguing actress. Her performances take one by surprise. She delivers no matter what role she is asked to give - from wacko in Stephen King's \"The Stand\" to her television performances. However, \"Checking Out\" allows her to shine. It is a role she is meant to play. The film is brilliantly directed by Jeff Hare. He was able to bring out the best in his cast and his direction - in every aspect - made the film a wonderful treasure. Jeff Hare was able to make a difficult theme laughable and yet profound. He gives us an up close and personal look at why indie films need to be made. The directors knowledge of his cast and script are extended to the finished film. The results are superb. Hopefully, it will be made available to large audiences because this is one you won't want to miss. It has the potential of being the sleeper hit of 2005 - in the fashion of \"My Big Fat Greek Wedding.\"\n",
      "label= 1\n",
      "17000\n",
      "guid= dev-17000\n",
      "text_a= One of the finest movies I have viewed...Good script, original plot of a man who is haunted about JFK's assassination when he was assigned to protect him on that Cold November day in 1963. Thirty years later another anti-social lunatic wants to assassinate the current president. The secret service agent loses his partner along the way,to the crazed gunmen who schemes,lies and murders anybody in his path who'll stand in his way of his mission.  The movie accompanies with a great memorable score,and a restrained but meaningful romance between Russo and Eastwood....which displays how difficult it is to have a romantic life in that kind of work. Malchovich is great,sure many other candidates could have played the role that he played,but how many could acted with such craftiness,and intellect that he displayed in the movie? Needless to say,I thought this was a great movie...everytime it's on television I have to watch it..and I own it on dvd! I'm a big Eastwood fan,this only boosted his already fabulous career,and Malchovich's best role to date! \n",
      "label= 1\n",
      "18000\n",
      "guid= dev-18000\n",
      "text_a= To remake \"Lost Horizon\", as a musical, the need for a Rodgers & Hammerstein or Lerner & Lowe type musical composition was needed. Burt Bacharach and Hal David were the wrong choice. Having said that, my favorite thing about \"Lost Horizon\", is its score. It's just that the score doesn't fit the piece. The cast, is made-up of mostly non-musical talents (Ullman, Finch and Hussey, were all dubbed, and still don't sound all that great). Frankly, the novella, on which this, and the earlier non-musical film versions were based, is mediocre, at best. While the possibilities for a truly good, cinematic musical version exist, they are not realized here. The film succeeds at being a good, rainy-day vehicle, to pass the time. Otherwise, you are better off, buying the CD of its soundtrack. Only recommended as a curiosity piece, due to the film's awful reputation. I've seen much better; but I've seen MUCH worse.\n",
      "label= 0\n",
      "19000\n",
      "guid= dev-19000\n",
      "text_a= Tourist Trap is an odd thriller that came out in the 70's, it's about 5 friends Molly (Jocelyn Jones), Jerry (Jon Van Hess), Eileen (Robin Sherwood), Becky (Tanya Roberts) and Woody (Kevin McDermott), who stumble upon a cloesd down museum SLAUSEN'S LOST OASIS, a curious and eerie roadside museum. This goldmine of decaying, but strangely life-like mannequins is run by Slausen (Chuck Conners), an eccentric, but seemingly harmless has-been. Slausen has one warning for the youngsters: Stay away from Davey, Slausen's reclusive and disturbed brother. The youngsters' curiosity gets the best of them and they go exploring. The trap is sprung! Amidst flying objects, slamming doors, scarves that strangle on their own, empowered by some hidden force, the trap slowly closes in on the group. The \"Creature\" Davey and his army of murderous mannequins make quick and brutal work of the friends, until only one remains. Although not a Slasher movie \"Tourist Trap\" still contains elements of slasher movies such as the chase scenes and the stalking and the fact the killer wears a mask. The seemingly telekinetic abilities of the killer to lock bolts and animate the wax dummies, is used to great effect. Perhaps the scariest thing about this movie are the mannequins, which are admittedly scary enough to start off with, but are rally spooky here. The film succeeds despite, or perhaps because of, an obviously meagre budget. These wax figures are blatantly plastic shop dummies- but this only goes to serve as even more eerie when their eyes move with an incredible human The acting is actually pretty good, Chuck Conners gives a well rounded and creepy performance as Mr Slausen, Jocelyn Jones is great as the female lead.\n",
      "label= 1\n",
      "20000\n",
      "guid= dev-20000\n",
      "text_a= I just had to add my comment to raise the average on this one. Paul Giamatti lets it all hang out in this one and is a hoot. He would probably say it was easy, but he really does a great job and should have won something for it. We've had the DVD for several years and my kids (boy now 4 1/2 and girl 9) will watch this one over and over, and the humor is adult enough that I don't mind having to hear it in the background (and I do run to the TV for the really funny parts). Simple moral message, lots of decent action and slapstick, \"bad\" grownups acting goofy to take the edge off, minimal bad language and minimal potty jokes make it hard to beat for a family standard.\n",
      "label= 1\n",
      "21000\n",
      "guid= dev-21000\n",
      "text_a= No wonder so many young people have Attention Deficit Disorder. It seems that stage (dance) productions these days are all about how many cameras and camera angles a director/ editor can squeeze into a 1 hour show. Is there a special Emmy category for this feat? Try counting them sometimes for something different to do with this, otherwise, completely unwatchable show.  I tried to make out at least a few faces of some of the other dancers in the production. That was impossible. They didn't appear to have any faces, just blurs - it was just Michael Flatley's face, Michael Flatley's bare chest(nice sheen!), Michael Flatley's feet, and that patented Flatley over-the-shoulder-come-hither look repeated infinity squared. Since he was an executive producer of this cut and paste job I guess that was to be expected. One doesn't have to wonder too much as to who his target audience is.  Riverdance was a much better production, as it tried to present the show pretty much as one might see it from the audience, not the catwalk,side wings, or floor nail perspective. If I'm not mistaken,I believe Sir Michael has retired. Thank God for small blessings.\n",
      "label= 0\n",
      "22000\n",
      "guid= dev-22000\n",
      "text_a= This \"film,\" and I use that term loosely, reminds me of the first joke my daughter wrote, at eighteen months: \"P.U., stinky poopies!\"  Like that joke, this movie can only appeal to the very young, the very immature, or the very stupid.  That said, there are a few bright spots.  The effects, where the majority of the reputed $100 million went, are kinetic and convincing -- I mean, as convincing as those kind of kinetic CGI effects can be. The CGI baby effects are not great, but I imagine those are very hard to do well... although for a hundred-million bucks, they could have been better! Moose, the dog from \"Frasier,\" phoned in his usual exemplary performance. Steven Wright did well with a small part. Alan Cummings was, well, Alan Cummings-as-villain, which we've seen before, and Bob Hoskins as Odin was unrecognizable, but enjoyable.  The actress playing Mrs. Avery was cute-as-a-button, as you'd expect, and Jamie Kennedy stunk, as you'd expect. His best role so far was in the Scream trilogy (not to be confused with the Lord of the Rings trilogy), and in Three Kings. He should stick, perhaps, to more subtle forms of comedy. Jim Carrey, he ain't. The writing and direction were, if anything, worse than Kennedy's performance. I semi-remember one clever (though seven-year-old clever) line that I wish someone would quote accurately for the \"Memorable Quotes\" section. Something about Avery's proposed costume being the \"crappiest crap in Craptown,\" it was a second-grade joke, but sort of funny in context. Over all, since there's nothing lower than a \"one,\" I give this film a \"one.\"\n",
      "label= 0\n",
      "23000\n",
      "guid= dev-23000\n",
      "text_a= Sure, this film was retarded. But you expected that the moment you looked at the cover-box. It's a B movie, and on the T&A factor this movie delivered. Truthfully, it was funnier than expected. While it was by no means a work of comedic genius, like \"The Party Animal\" or \"Orgazmo\", as far as B movies go it was worth the watch, if you're into that sort of thing anyway. Christians and morally-oriented parental groups, this is soft-core adult entertainment. If you don't want your children watching sexual content and nudity, then you should keep your children away from this film.\n",
      "label= 0\n",
      "24000\n",
      "guid= dev-24000\n",
      "text_a= Let me tell you a story. One day on the streets of Athens a film director bumped into a male prostitute and decided that the world just HAD to know his story because...you know... he's deprived...and he takes his shirt off a lot and...so on. This film is the result of his revelation. Repulsive, depraved, homophobic, misogynist...but of course filled with pretty guys with their chests showing. If this is your idea of a good film then enjoy, if not avoid it like the plague. It's put me off ever going to Greece that's for sure.\n",
      "label= 0\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "05/25/2024 20:02:15 - INFO - __main__ -   ***** Running training *****\n",
      "05/25/2024 20:02:15 - INFO - __main__ -     Num examples = 25000\n",
      "05/25/2024 20:02:15 - INFO - __main__ -     Batch size = 6\n",
      "05/25/2024 20:02:15 - INFO - __main__ -     Num steps = 12500\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "evaluating steps: 625\n",
      "[0, 1, 2, 3]\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Epoch:   0%|          | 0/3 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Seeded everything\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "c:\\Users\\29303\\Downloads\\dml\\mytest\\optimization_lr.py:172: UserWarning: This overload of add_ is deprecated:\n",
      "\tadd_(Number alpha, Tensor other)\n",
      "Consider using one of the following signatures instead:\n",
      "\tadd_(Tensor other, *, Number alpha) (Triggered internally at ..\\torch\\csrc\\utils\\python_arg_parser.cpp:1519.)\n",
      "  next_m.mul_(beta1).add_(1 - beta1, grad)\n",
      "05/25/2024 21:38:15 - INFO - __main__ -   Start evaluating!\n",
      "Iteration:  15%|█▍        | 624/4167 [2:04:09<11:44:59, 11.94s/it]\n",
      "Epoch:   0%|          | 0/3 [2:04:09<?, ?it/s]\n"
     ]
    },
    {
     "ename": "KeyboardInterrupt",
     "evalue": "",
     "output_type": "error",
     "traceback": [
      "\u001b[1;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[1;31mKeyboardInterrupt\u001b[0m                         Traceback (most recent call last)",
      "Cell \u001b[1;32mIn[6], line 387\u001b[0m\n\u001b[0;32m    383\u001b[0m                 eval_step \u001b[38;5;241m+\u001b[39m\u001b[38;5;241m=\u001b[39m \u001b[38;5;241m1\u001b[39m\n\u001b[0;32m    384\u001b[0m                 global_step \u001b[38;5;241m+\u001b[39m\u001b[38;5;241m=\u001b[39m \u001b[38;5;241m1\u001b[39m   \n\u001b[1;32m--> 387\u001b[0m main()\n",
      "Cell \u001b[1;32mIn[6], line 337\u001b[0m, in \u001b[0;36mmain\u001b[1;34m()\u001b[0m\n\u001b[0;32m    332\u001b[0m state \u001b[38;5;241m=\u001b[39m {\n\u001b[0;32m    333\u001b[0m \u001b[38;5;124m'\u001b[39m\u001b[38;5;124mepoch\u001b[39m\u001b[38;5;124m'\u001b[39m: epoch\u001b[38;5;241m+\u001b[39m\u001b[38;5;241m1\u001b[39m,\n\u001b[0;32m    334\u001b[0m \u001b[38;5;124m'\u001b[39m\u001b[38;5;124mstate_dict\u001b[39m\u001b[38;5;124m'\u001b[39m: model\u001b[38;5;241m.\u001b[39mstate_dict(),\n\u001b[0;32m    335\u001b[0m }\n\u001b[0;32m    336\u001b[0m torch\u001b[38;5;241m.\u001b[39msave(state, args\u001b[38;5;241m.\u001b[39moutput_dir \u001b[38;5;241m+\u001b[39m \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124m/checkpoint-\u001b[39m\u001b[38;5;132;01m{}\u001b[39;00m\u001b[38;5;124m.pth.tar\u001b[39m\u001b[38;5;124m\"\u001b[39m\u001b[38;5;241m.\u001b[39mformat(\u001b[38;5;28mint\u001b[39m(global_step \u001b[38;5;241m/\u001b[39m args\u001b[38;5;241m.\u001b[39mgrad_eval_step))) \n\u001b[1;32m--> 337\u001b[0m eval_model(model, args, eval_dataloader, device, epoch, tr_loss, nb_tr_steps, global_step \u001b[38;5;241m/\u001b[39m args\u001b[38;5;241m.\u001b[39mgrad_eval_step)\n\u001b[0;32m    338\u001b[0m \u001b[38;5;66;03m# Calculate gradient changing ratio\u001b[39;00m\n\u001b[0;32m    339\u001b[0m \u001b[38;5;28;01mfor\u001b[39;00m name \u001b[38;5;129;01min\u001b[39;00m grad_tensor_dict\u001b[38;5;241m.\u001b[39mkeys():\n",
      "Cell \u001b[1;32mIn[4], line 66\u001b[0m, in \u001b[0;36meval_model\u001b[1;34m(model, args, eval_dataloader, device, epoch, tr_loss, nb_tr_steps, eval_step)\u001b[0m\n\u001b[0;32m     63\u001b[0m \u001b[38;5;28;01mwith\u001b[39;00m torch\u001b[38;5;241m.\u001b[39mno_grad():\n\u001b[0;32m     64\u001b[0m     (tmp_eval_loss, logits), _ \u001b[38;5;241m=\u001b[39m model(input_ids, segment_ids, input_mask, label_ids)\n\u001b[1;32m---> 66\u001b[0m logits \u001b[38;5;241m=\u001b[39m logits\u001b[38;5;241m.\u001b[39mdetach()\u001b[38;5;241m.\u001b[39mcpu()\u001b[38;5;241m.\u001b[39mnumpy()\n\u001b[0;32m     67\u001b[0m label_ids \u001b[38;5;241m=\u001b[39m label_ids\u001b[38;5;241m.\u001b[39mto(\u001b[38;5;124m'\u001b[39m\u001b[38;5;124mcpu\u001b[39m\u001b[38;5;124m'\u001b[39m)\u001b[38;5;241m.\u001b[39mnumpy()\n\u001b[0;32m     68\u001b[0m outputs \u001b[38;5;241m=\u001b[39m np\u001b[38;5;241m.\u001b[39margmax(logits, axis\u001b[38;5;241m=\u001b[39m\u001b[38;5;241m1\u001b[39m)\n",
      "\u001b[1;31mKeyboardInterrupt\u001b[0m: "
     ]
    }
   ],
   "source": [
    "def main():\n",
    "    parser = argparse.ArgumentParser()\n",
    "\n",
    "    ## Required parameters\n",
    "    parser.add_argument(\"--data_dir\",\n",
    "                        default=\"IMDB_data\",\n",
    "                        type=str,\n",
    "                        #required=True,\n",
    "                        help=\"The input data dir. Should contain the .tsv files (or other data files) for the task.\")\n",
    "    parser.add_argument(\"--bert_config_file\",\n",
    "                        default=\"uncased_L-12_H-768_A-12/bert_config.json\",\n",
    "                        type=str,\n",
    "                        #required=True,\n",
    "                        help=\"The config json file corresponding to the pre-trained BERT model. \\n\"\n",
    "                             \"This specifies the model architecture.\")\n",
    "    parser.add_argument(\"--task_name\",\n",
    "                        default=\"imdb\",\n",
    "                        type=str,\n",
    "                        #required=True,\n",
    "                        help=\"The name of the task to train.\")\n",
    "    parser.add_argument(\"--vocab_file\",\n",
    "                        default=\"uncased_L-12_H-768_A-12/vocab.txt\",\n",
    "                        type=str,\n",
    "                        #required=True,\n",
    "                        help=\"The vocabulary file that the BERT model was trained on.\")\n",
    "    parser.add_argument(\"--output_dir\",\n",
    "                        default=\"output\",\n",
    "                        type=str,\n",
    "                        #required=True,\n",
    "                        help=\"The output directory where the model checkpoints will be written.\")\n",
    "\n",
    "    ## Other parameters\n",
    "    parser.add_argument(\"--init_checkpoint\",\n",
    "                        default=\"uncased_L-12_H-768_A-12/pytorch_model.bin\",\n",
    "                        type=str,\n",
    "                        help=\"Initial checkpoint (usually from a pre-trained BERT model).\")\n",
    "    parser.add_argument(\"--do_lower_case\",\n",
    "                        default=False,\n",
    "                        action='store_true',\n",
    "                        help=\"Whether to lower case the input text. True for uncased models, False for cased models.\")\n",
    "    parser.add_argument(\"--max_seq_length\",\n",
    "                        default=512,\n",
    "                        type=int,\n",
    "                        help=\"The maximum total input sequence length after WordPiece tokenization. \\n\"\n",
    "                             \"Sequences longer than this will be truncated, and sequences shorter \\n\"\n",
    "                             \"than this will be padded.\")\n",
    "    parser.add_argument(\"--do_train\",\n",
    "                        default=True,\n",
    "                        action='store_true',\n",
    "                        help=\"Whether to run training.\")\n",
    "    parser.add_argument(\"--do_eval\",\n",
    "                        default=True,\n",
    "                        action='store_true',\n",
    "                        help=\"Whether to run eval on the dev set.\")\n",
    "    parser.add_argument(\"--discr\",\n",
    "                        default=False,\n",
    "                        action='store_true',\n",
    "                        help=\"Whether to do discriminative fine-tuning.\")\n",
    "    parser.add_argument(\"--train_batch_size\",\n",
    "                        default=6,\n",
    "                        type=int,\n",
    "                        help=\"Total batch size for training.\")\n",
    "    parser.add_argument(\"--eval_batch_size\",\n",
    "                        default=8,\n",
    "                        type=int,\n",
    "                        help=\"Total batch size for eval.\")\n",
    "    parser.add_argument(\"--learning_rate\",\n",
    "                        default=1e-5,\n",
    "                        type=float,\n",
    "                        help=\"The initial learning rate for Adam.\")\n",
    "    parser.add_argument(\"--num_train_epochs\",\n",
    "                        default=3.0,\n",
    "                        type=float,\n",
    "                        help=\"Total number of training epochs to perform.\")\n",
    "    parser.add_argument(\"--warmup_proportion\",\n",
    "                        default=0.1,\n",
    "                        type=float,\n",
    "                        help=\"Proportion of training to perform linear learning rate warmup for. \"\n",
    "                             \"E.g., 0.1 = 10%% of training.\")\n",
    "    parser.add_argument(\"--save_checkpoints_steps\",\n",
    "                        default=1000,\n",
    "                        type=int,\n",
    "                        help=\"How often to save the model checkpoint.\")\n",
    "    parser.add_argument(\"--no_cuda\",\n",
    "                        default=False,\n",
    "                        action='store_true',\n",
    "                        help=\"Whether not to use CUDA when available\")\n",
    "    parser.add_argument(\"--accumulate_gradients\",\n",
    "                        type=int,\n",
    "                        default=1,\n",
    "                        help=\"Number of steps to accumulate gradient on (divide the batch_size and accumulate)\")\n",
    "    parser.add_argument(\"--local_rank\",\n",
    "                        type=int,\n",
    "                        default=-1,\n",
    "                        help=\"local_rank for distributed training on gpus\")\n",
    "    parser.add_argument('--seed', \n",
    "                        type=int, \n",
    "                        default=42,\n",
    "                        help=\"random seed for initialization\")\n",
    "    parser.add_argument('--gradient_accumulation_steps',\n",
    "                        type=int,\n",
    "                        default=1,\n",
    "                        help=\"Number of updates steps to accumualte before performing a backward/update pass.\")\n",
    "    \n",
    "    parser.add_argument('--grad_eval_step',\n",
    "                        type=int,\n",
    "                        default=500,\n",
    "                        help=\"Number of iterations for one evaluation interval.\")  \n",
    "    parser.add_argument('--num_datas',\n",
    "                        type=int,\n",
    "                        default=None,\n",
    "                        help=\"Number of data points.\")     \n",
    "    parser.add_argument('--num_training_data',\n",
    "                        type=int,\n",
    "                        default=60000,\n",
    "                        help=\"\")\n",
    "    parser.add_argument('--num_intervals',\n",
    "                        type=int,\n",
    "                        default=20,\n",
    "                        help=\"Number of evaluation intervals.\")\n",
    "    parser.add_argument('--decay_factor',\n",
    "                        type=int,\n",
    "                        default=10,\n",
    "                        help=\"Decay factor for learning rate scheduling.\") \n",
    "    \n",
    "    parser.add_argument('--percentile',\n",
    "                        type=int,\n",
    "                        default=50,\n",
    "                        help=\"Percentile for freezing. \")\n",
    "    parser.add_argument('--random_seeds',\n",
    "                        type=str,\n",
    "                        default=\"0,1,2,3\",\n",
    "                        help=\"Random seeds for each training epoch, separated by comma.\")\n",
    "                                 \n",
    "    #args = parser.parse_args()\n",
    "    args, unknown = parser.parse_known_args() \n",
    "\n",
    "    processors = {\n",
    "        \"ag\": AGNewsProcessor,\n",
    "        \"ag_sep\": AGNewsProcessor_sep,\n",
    "        \"ag_sep_aug\": AGNewsProcessor_sep_aug,\n",
    "        \"imdb\": IMDBProcessor,\n",
    "        \"imdb_sep\": IMDBProcessor_sep,\n",
    "        \"imdb_sep_aug\": IMDBProcessor_sep_aug,\n",
    "        \"yelp_p\": Yelp_p_Processor,\n",
    "        \"yelp_f\": Yelp_f_Processor,\n",
    "        \"yahoo\": Yahoo_Processor,\n",
    "        \"trec\": Trec_Processor,\n",
    "        \"dbpedia\":Dbpedia_Processor,\n",
    "        \"mrpc\": MrpcProcessor,\n",
    "        \"sogou\": Sogou_Processor,\n",
    "        \"cola\": ColaProcessor\n",
    "    }\n",
    "\n",
    "    if args.local_rank == -1 or args.no_cuda:\n",
    "        device = torch.device(\"cuda\" if torch.cuda.is_available() and not args.no_cuda else \"cpu\")\n",
    "        n_gpu = torch.cuda.device_count()\n",
    "    else:\n",
    "        device = torch.device(\"cuda\", args.local_rank)\n",
    "        n_gpu = 1\n",
    "        # Initializes the distributed backend which will take care of sychronizing nodes/GPUs\n",
    "        torch.distributed.init_process_group(backend='nccl')\n",
    "    logger.info(\"device %s n_gpu %d distributed training %r\", device, n_gpu, bool(args.local_rank != -1))\n",
    "\n",
    "    if args.accumulate_gradients < 1:\n",
    "        raise ValueError(\"Invalid accumulate_gradients parameter: {}, should be >= 1\".format(\n",
    "                            args.accumulate_gradients))\n",
    "\n",
    "    args.train_batch_size = int(args.train_batch_size / args.accumulate_gradients)\n",
    "\n",
    "    if not args.do_train and not args.do_eval:\n",
    "        raise ValueError(\"At least one of `do_train` or `do_eval` must be True.\")\n",
    "\n",
    "    bert_config = BertConfig.from_json_file(args.bert_config_file)\n",
    "\n",
    "    if args.max_seq_length > bert_config.max_position_embeddings:\n",
    "        raise ValueError(\n",
    "            \"Cannot use sequence length {} because the BERT model was only trained up to sequence length {}\".format(\n",
    "            args.max_seq_length, bert_config.max_position_embeddings))\n",
    "\n",
    "    if os.path.exists(args.output_dir) and os.listdir(args.output_dir):\n",
    "        raise ValueError(\"Output directory ({}) already exists and is not empty.\".format(args.output_dir))\n",
    "    os.makedirs(args.output_dir, exist_ok=True)\n",
    "\n",
    "    task_name = args.task_name.lower()\n",
    "\n",
    "    if task_name not in processors:\n",
    "        raise ValueError(\"Task not found: %s\" % (task_name))\n",
    "\n",
    "    processor = processors[task_name]()\n",
    "    label_list = processor.get_labels()\n",
    "\n",
    "    tokenizer = tokenization.FullTokenizer(\n",
    "        vocab_file=args.vocab_file, do_lower_case=args.do_lower_case)\n",
    "\n",
    "    train_examples = None\n",
    "    num_train_steps = None\n",
    "    if args.do_train:\n",
    "        train_examples = processor.get_train_examples(args.data_dir, data_num = args.num_datas)\n",
    "        num_train_steps = int(\n",
    "            len(train_examples) / args.train_batch_size * args.num_train_epochs)\n",
    "    set_seed(0)\n",
    "    model = BertForSequenceClassification(bert_config, len(label_list))\n",
    "    if args.init_checkpoint is not None:\n",
    "        model.bert.load_state_dict(torch.load(args.init_checkpoint, map_location='cpu'))\n",
    "    model.to(device)\n",
    "    # for name, param in model.bert.named_parameters():\n",
    "    #     print(param.shape)\n",
    "    if args.local_rank != -1:\n",
    "        model = torch.nn.parallel.DistributedDataParallel(model, device_ids=[args.local_rank],\n",
    "                                                          output_device=args.local_rank)\n",
    "    elif n_gpu > 1:\n",
    "        model = torch.nn.DataParallel(model)\n",
    "\n",
    "    no_decay = ['bias', 'gamma', 'beta']\n",
    "    if args.discr:\n",
    "        group1=['layer.0.','layer.1.','layer.2.','layer.3.']\n",
    "        group2=['layer.4.','layer.5.','layer.6.','layer.7.']\n",
    "        group3=['layer.8.','layer.9.','layer.10.','layer.11.']\n",
    "        group_all=['layer.0.','layer.1.','layer.2.','layer.3.','layer.4.','layer.5.','layer.6.','layer.7.','layer.8.','layer.9.','layer.10.','layer.11.']\n",
    "        optimizer_parameters = [\n",
    "            {'params': [p for n, p in model.named_parameters() if not any(nd in n for nd in no_decay) and not any(nd in n for nd in group_all)],'weight_decay_rate': 0.01},\n",
    "            \n",
    "            {'params': [p for n, p in model.named_parameters() if not any(nd in n for nd in no_decay) and any(nd in n for nd in group1)],'weight_decay_rate': 0.01, 'lr': args.learning_rate/2.6},\n",
    "            \n",
    "            {'params': [p for n, p in model.named_parameters() if not any(nd in n for nd in no_decay) and any(nd in n for nd in group2)],'weight_decay_rate': 0.01, 'lr': args.learning_rate},\n",
    "            \n",
    "            {'params': [p for n, p in model.named_parameters() if not any(nd in n for nd in no_decay) and any(nd in n for nd in group3)],'weight_decay_rate': 0.01, 'lr': args.learning_rate*2.6},\n",
    "            \n",
    "            {'params': [p for n, p in model.named_parameters() if any(nd in n for nd in no_decay) and not any(nd in n for nd in group_all)],'weight_decay_rate': 0.0},\n",
    "            \n",
    "            {'params': [p for n, p in model.named_parameters() if any(nd in n for nd in no_decay) and any(nd in n for nd in group1)],'weight_decay_rate': 0.0, 'lr': args.learning_rate/2.6},\n",
    "            \n",
    "            {'params': [p for n, p in model.named_parameters() if any(nd in n for nd in no_decay) and any(nd in n for nd in group2)],'weight_decay_rate': 0.0, 'lr': args.learning_rate},\n",
    "            {'params': [p for n, p in model.named_parameters() if any(nd in n for nd in no_decay) and any(nd in n for nd in group3)],'weight_decay_rate': 0.0, 'lr': args.learning_rate*2.6},\n",
    "        ]\n",
    "    else:\n",
    "        optimizer_parameters = [\n",
    "             {'params': [p for n, p in model.named_parameters() if not any(nd in n for nd in no_decay)], 'weight_decay_rate': 0.01},\n",
    "             {'params': [p for n, p in model.named_parameters() if any(nd in n for nd in no_decay)], 'weight_decay_rate': 0.0}\n",
    "             ]\n",
    "\t\t\n",
    "    optimizer = BERTAdam(optimizer_parameters,\n",
    "                         lr=args.learning_rate,\n",
    "                         warmup=args.warmup_proportion,\n",
    "                         t_total=num_train_steps,\n",
    "                         decay_factor=args.decay_factor)\n",
    "\t\n",
    "    eval_examples = processor.get_dev_examples(args.data_dir)\n",
    "    eval_features = convert_examples_to_features(\n",
    "        eval_examples, label_list, args.max_seq_length, tokenizer)\n",
    "\n",
    "    all_input_ids = torch.tensor([f.input_ids for f in eval_features], dtype=torch.long)\n",
    "    all_input_mask = torch.tensor([f.input_mask for f in eval_features], dtype=torch.long)\n",
    "    all_segment_ids = torch.tensor([f.segment_ids for f in eval_features], dtype=torch.long)\n",
    "    all_label_ids = torch.tensor([f.label_id for f in eval_features], dtype=torch.long)\n",
    "\n",
    "    eval_data = TensorDataset(all_input_ids, all_input_mask, all_segment_ids, all_label_ids)\n",
    "    eval_dataloader = DataLoader(eval_data, batch_size=args.eval_batch_size, shuffle=False, num_workers=0)\n",
    "    global global_step\n",
    "    global nb_tr_examples, nb_tr_steps, tr_loss    \n",
    "    if args.do_train:\n",
    "        train_features = convert_examples_to_features(\n",
    "            train_examples, label_list, args.max_seq_length, tokenizer)\n",
    "        logger.info(\"***** Running training *****\")\n",
    "        logger.info(\"  Num examples = %d\", len(train_examples))\n",
    "        logger.info(\"  Batch size = %d\", args.train_batch_size)\n",
    "        logger.info(\"  Num steps = %d\", num_train_steps)\n",
    "\n",
    "        all_input_ids = torch.tensor([f.input_ids for f in train_features], dtype=torch.long)\n",
    "        all_input_mask = torch.tensor([f.input_mask for f in train_features], dtype=torch.long)\n",
    "        all_segment_ids = torch.tensor([f.segment_ids for f in train_features], dtype=torch.long)\n",
    "        all_label_ids = torch.tensor([f.label_id for f in train_features], dtype=torch.long)\n",
    "\n",
    "        train_data = TensorDataset(all_input_ids, all_input_mask, all_segment_ids, all_label_ids)\n",
    "        if args.local_rank == -1:\n",
    "            train_sampler = RandomSampler(range(0, min(args.num_training_data, len(train_data))))\n",
    "        else:\n",
    "            train_sampler = DistributedSampler(train_data)\n",
    "        train_dataloader = DataLoader(train_data, sampler=train_sampler, batch_size=args.train_batch_size, num_workers=0)\n",
    "        epoch=0\n",
    "        start_layer = 0\n",
    "        prev_intermediate_grad_dict = None\n",
    "        grad_tensor_dict = {}\n",
    "        for name, param in model.bert.named_parameters():\n",
    "            grad_tensor_dict[name] = torch.zeros(param.shape).to(device)\n",
    "        args.grad_eval_step = int(args.num_train_epochs * len(train_dataloader)* (1/args.num_intervals)) \n",
    "        print(\"evaluating steps:\", args.grad_eval_step)\n",
    "        seeds = [int(seed) for seed in args.random_seeds.split(',')]\n",
    "        print(seeds)\n",
    "        eval_step = 0\n",
    "        for _ in trange(int(args.num_train_epochs), desc=\"Epoch\"):\n",
    "            epoch+=1\n",
    "            \n",
    "            \n",
    "            tr_loss = 0\n",
    "            nb_tr_examples, nb_tr_steps = 0, 0\n",
    "            set_seed(seeds[epoch-1])\n",
    "            for step, batch in enumerate(tqdm(train_dataloader, desc=\"Iteration\")):\n",
    "                batch = tuple(t.to(device) for t in batch)\n",
    "                model.train()\n",
    "                input_ids, input_mask, segment_ids, label_ids = batch\n",
    "                \n",
    "                (loss,_), _ = model(input_ids, segment_ids, input_mask, label_ids, start_layer=start_layer)\n",
    "                if args.gradient_accumulation_steps > 1:\n",
    "                    loss = loss / args.gradient_accumulation_steps\n",
    "                loss.backward()\n",
    "                tr_loss += loss.item()\n",
    "                # print(loss.item())\n",
    "                nb_tr_examples += input_ids.size(0)\n",
    "                nb_tr_steps += 1\n",
    "                # Accumulate gradient vector for this interval\n",
    "                for name, param in model.bert.named_parameters():\n",
    "                    if param.grad is not None:\n",
    "                        \n",
    "                        if name not in grad_tensor_dict.keys():\n",
    "                            grad_tensor_dict[name] = param.grad\n",
    "                        else:\n",
    "                            grad_tensor_dict[name] += param.grad\n",
    "                current_grad_dict = {}\n",
    "                for i in range(12):\n",
    "                    current_grad_dict[i] = 0\n",
    "                \n",
    "                if (step + 1) % args.gradient_accumulation_steps == 0:\n",
    "                    optimizer.step(current_step=global_step)    # We have accumulated enought gradients\n",
    "                    model.zero_grad()\n",
    "                \n",
    "                if (eval_step + 1) % args.grad_eval_step == 0 and eval_step / args.grad_eval_step > 0:\n",
    "                    \n",
    "                    logger.info(\"Start evaluating!\")\n",
    "                    # Saving checkpoints\n",
    "                    state = {\n",
    "                    'epoch': epoch+1,\n",
    "                    'state_dict': model.state_dict(),\n",
    "                    }\n",
    "                    torch.save(state, args.output_dir + \"/checkpoint-{}.pth.tar\".format(int(global_step / args.grad_eval_step))) \n",
    "                    eval_model(model, args, eval_dataloader, device, epoch, tr_loss, nb_tr_steps, global_step / args.grad_eval_step)\n",
    "                    # Calculate gradient changing ratio\n",
    "                    for name in grad_tensor_dict.keys():\n",
    "                        param_list = name.split(\".\")\n",
    "                        layer_num = 0\n",
    "                        for split_param in param_list:\n",
    "                            try:\n",
    "                                layer_num = int(split_param)\n",
    "                                if \"encoder\" in name:\n",
    "                                    current_grad_dict[layer_num] += torch.norm(grad_tensor_dict[name].cpu().detach(), p=1).item() \n",
    "                            except ValueError:\n",
    "                                pass\n",
    "                        \n",
    "                    print(current_grad_dict)                    \n",
    "                    print(\"grad dict\", current_grad_dict) \n",
    "                    # Clear gradient accumulator\n",
    "                    grad_tensor_dict = {}\n",
    "                    for name, param in model.bert.named_parameters():\n",
    "                        grad_tensor_dict[name] = torch.zeros(param.shape).to(device)\n",
    "\n",
    "                    if prev_intermediate_grad_dict is None:\n",
    "                        # Set gradient dict to be compared with for the first time\n",
    "                        prev_intermediate_grad_dict = current_grad_dict\n",
    "                    else:\n",
    "                        threshold_dict = {}\n",
    "                        for key in range(12):\n",
    "                            threshold_dict[key] = 0\n",
    "                        # Calculate gradient changing threshold\n",
    "                        for key in current_grad_dict.keys() :\n",
    "                            if current_grad_dict[key] > 0:\n",
    "                                threshold_dict[key] = abs(prev_intermediate_grad_dict[key] - current_grad_dict[key]) / prev_intermediate_grad_dict[key]    \n",
    "                            \n",
    "                        median_value = np.percentile(list(threshold_dict.values())[start_layer:], args.percentile)   \n",
    "                        # Find out the first layer with ratio ge to the median value      \n",
    "                        for key in threshold_dict.keys():\n",
    "                            if threshold_dict[key] >= median_value:\n",
    "                                start_layer = key\n",
    "                                break\n",
    "                        prev_intermediate_grad_dict = current_grad_dict\n",
    "                        print(\"threshold: \", threshold_dict)\n",
    "                        print(\"layer num: \", start_layer)\n",
    "                        if start_layer > 0 :\n",
    "                            # New optimizer \n",
    "                            \n",
    "                            model, optimizer = frozen_model(bert_config, label_list, args, global_step, num_train_steps, device)\n",
    "                        logger.info(\"TRAINING FROM {}\".format(str(start_layer)))\n",
    "                eval_step += 1\n",
    "                global_step += 1   \n",
    "\n",
    "\n",
    "main()"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "base",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.11.4"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}