diff --git a/.travis.yml b/.travis.yml index a16f6187..0cb9e0fc 100644 --- a/.travis.yml +++ b/.travis.yml @@ -3,6 +3,7 @@ language: python python: - "3.6" - "3.7" + - "3.8" install: - pip install . flake8 mypy pylint pytest script: @@ -10,3 +11,6 @@ script: - mypy annofabcli --config-file setup.cfg - pylint annofabcli --rcfile setup.cfg - pytest tests/test_local*.py +branches: + only: + - master diff --git a/Pipfile.lock b/Pipfile.lock index cd548028..2d60b611 100644 --- a/Pipfile.lock +++ b/Pipfile.lock @@ -17,10 +17,10 @@ "develop": { "annofabapi": { "hashes": [ - "sha256:9724d9161f9d70c6e2d2a36262db410c66317af198abad286e236bb7f1e4d15f", - "sha256:b079853ce9ee173d71ba3661ffbcb68ebe36af7ac68f4f8471eedf9bedde36e3" + "sha256:931f306eadeee3f96ce38b91362d9b97ab289d8cd619a07e398725728036f768", + "sha256:a84a621b39f0705a5da8a43ace9b3263febe8262fa784e9ba3447ec0f134d280" ], - "version": "==0.20.0" + "version": "==0.20.1" }, "annofabcli": { "editable": true, @@ -28,10 +28,10 @@ }, "astroid": { "hashes": [ - "sha256:98c665ad84d10b18318c5ab7c3d203fe11714cbad2a4aef4f44651f415392754", - "sha256:b7546ffdedbf7abcfbff93cd1de9e9980b1ef744852689decc5aeada324238c6" + "sha256:09a3fba616519311f1af8a461f804b68f0370e100c9264a035aa7846d7852e33", + "sha256:5a79c9b4bd6c4be777424593f957c996e20beb5f74e0bc332f47713c6f675efe" ], - "version": "==2.3.1" + "version": "==2.3.2" }, "atomicwrites": { "hashes": [ @@ -127,10 +127,10 @@ }, "dataclasses-json": { "hashes": [ - "sha256:4040d1c84061929fc79a53b2d1816249d7468e8a84d1a776b33421f38e71a08c", - "sha256:71eb3ba2f133d2380c12d79a4b622f6a95395e0aa0ab52ef2f8043d8bf2e3d4c" + "sha256:3f348a132c6c84772b99fca50c447ef3b8382d274fd9a539c958dd9b93ba4806", + "sha256:8851be971187d22a898247ffff9b23de6d5d1db93b8e648997a71a2e4023d13c" ], - "version": "==0.3.4" + "version": "==0.3.5" }, "dictdiffer": { "hashes": [ @@ -327,47 +327,56 @@ }, "mypy": { "hashes": [ - "sha256:1d98fd818ad3128a5408148c9e4a5edce6ed6b58cc314283e631dd5d9216527b", - "sha256:22ee018e8fc212fe601aba65d3699689dd29a26410ef0d2cc1943de7bec7e3ac", - "sha256:3a24f80776edc706ec8d05329e854d5b9e464cd332e25cde10c8da2da0a0db6c", - "sha256:42a78944e80770f21609f504ca6c8173f7768043205b5ac51c9144e057dcf879", - "sha256:4b2b20106973548975f0c0b1112eceb4d77ed0cafe0a231a1318f3b3a22fc795", - "sha256:591a9625b4d285f3ba69f541c84c0ad9e7bffa7794da3fa0585ef13cf95cb021", - "sha256:5b4b70da3d8bae73b908a90bb2c387b977e59d484d22c604a2131f6f4397c1a3", - "sha256:84edda1ffeda0941b2ab38ecf49302326df79947fa33d98cdcfbf8ca9cf0bb23", - "sha256:b2b83d29babd61b876ae375786960a5374bba0e4aba3c293328ca6ca5dc448dd", - "sha256:cc4502f84c37223a1a5ab700649b5ab1b5e4d2bf2d426907161f20672a21930b", - "sha256:e29e24dd6e7f39f200a5bb55dcaa645d38a397dd5a6674f6042ef02df5795046" + "sha256:1521c186a3d200c399bd5573c828ea2db1362af7209b2adb1bb8532cea2fb36f", + "sha256:31a046ab040a84a0fc38bc93694876398e62bc9f35eca8ccbf6418b7297f4c00", + "sha256:3b1a411909c84b2ae9b8283b58b48541654b918e8513c20a400bb946aa9111ae", + "sha256:48c8bc99380575deb39f5d3400ebb6a8a1cb5cc669bbba4d3bb30f904e0a0e7d", + "sha256:540c9caa57a22d0d5d3c69047cc9dd0094d49782603eb03069821b41f9e970e9", + "sha256:672e418425d957e276c291930a3921b4a6413204f53fe7c37cad7bc57b9a3391", + "sha256:6ed3b9b3fdc7193ea7aca6f3c20549b377a56f28769783a8f27191903a54170f", + "sha256:9371290aa2cad5ad133e4cdc43892778efd13293406f7340b9ffe99d5ec7c1d9", + "sha256:ace6ac1d0f87d4072f05b5468a084a45b4eda970e4d26704f201e06d47ab2990", + "sha256:b428f883d2b3fe1d052c630642cc6afddd07d5cd7873da948644508be3b9d4a7", + "sha256:d5bf0e6ec8ba346a2cf35cb55bf4adfddbc6b6576fcc9e10863daa523e418dbb", + "sha256:d7574e283f83c08501607586b3167728c58e8442947e027d2d4c7dcd6d82f453", + "sha256:dc889c84241a857c263a2b1cd1121507db7d5b5f5e87e77147097230f374d10b", + "sha256:f4748697b349f373002656bf32fede706a0e713d67bfdcf04edf39b1f61d46eb" ], "index": "pypi", - "version": "==0.730" + "version": "==0.740" }, "mypy-extensions": { "hashes": [ - "sha256:a161e3b917053de87dbe469987e173e49fb454eca10ef28b48b384538cc11458" + "sha256:090fedd75945a69ae91ce1303b5824f428daf5a028d2f6ab8a299250a846f15d", + "sha256:2d82818f5bb3e369420cb3c4060a7970edba416647068eb4c5343488a6c604a8" ], - "version": "==0.4.2" + "version": "==0.4.3" }, "numpy": { "hashes": [ - "sha256:05dbfe72684cc14b92568de1bc1f41e5f62b00f714afc9adee42f6311738091f", - "sha256:0d82cb7271a577529d07bbb05cb58675f2deb09772175fab96dc8de025d8ac05", - "sha256:10132aa1fef99adc85a905d82e8497a580f83739837d7cbd234649f2e9b9dc58", - "sha256:12322df2e21f033a60c80319c25011194cd2a21294cc66fee0908aeae2c27832", - "sha256:16f19b3aa775dddc9814e02a46b8e6ae6a54ed8cf143962b4e53f0471dbd7b16", - "sha256:3d0b0989dd2d066db006158de7220802899a1e5c8cf622abe2d0bd158fd01c2c", - "sha256:438a3f0e7b681642898fd7993d38e2bf140a2d1eafaf3e89bb626db7f50db355", - "sha256:5fd214f482ab53f2cea57414c5fb3e58895b17df6e6f5bca5be6a0bb6aea23bb", - "sha256:73615d3edc84dd7c4aeb212fa3748fb83217e00d201875a47327f55363cef2df", - "sha256:7bd355ad7496f4ce1d235e9814ec81ee3d28308d591c067ce92e49f745ba2c2f", - "sha256:7d077f2976b8f3de08a0dcf5d72083f4af5411e8fddacd662aae27baa2601196", - "sha256:a4092682778dc48093e8bda8d26ee8360153e2047826f95a3f5eae09f0ae3abf", - "sha256:b458de8624c9f6034af492372eb2fee41a8e605f03f4732f43fc099e227858b2", - "sha256:e70fc8ff03a961f13363c2c95ef8285e0cf6a720f8271836f852cc0fa64e97c8", - "sha256:ee8e9d7cad5fe6dde50ede0d2e978d81eafeaa6233fb0b8719f60214cf226578", - "sha256:f4a4f6aba148858a5a5d546a99280f71f5ee6ec8182a7d195af1a914195b21a2" - ], - "version": "==1.17.2" + "sha256:0b0dd8f47fb177d00fa6ef2d58783c4f41ad3126b139c91dd2f7c4b3fdf5e9a5", + "sha256:25ffe71f96878e1da7e014467e19e7db90ae7d4e12affbc73101bcf61785214e", + "sha256:26efd7f7d755e6ca966a5c0ac5a930a87dbbaab1c51716ac26a38f42ecc9bc4b", + "sha256:28b1180c758abf34a5c3fea76fcee66a87def1656724c42bb14a6f9717a5bdf7", + "sha256:2e418f0a59473dac424f888dd57e85f77502a593b207809211c76e5396ae4f5c", + "sha256:30c84e3a62cfcb9e3066f25226e131451312a044f1fe2040e69ce792cb7de418", + "sha256:4650d94bb9c947151737ee022b934b7d9a845a7c76e476f3e460f09a0c8c6f39", + "sha256:4dd830a11e8724c9c9379feed1d1be43113f8bcce55f47ea7186d3946769ce26", + "sha256:4f2a2b279efde194877aff1f76cf61c68e840db242a5c7169f1ff0fd59a2b1e2", + "sha256:62d22566b3e3428dfc9ec972014c38ed9a4db4f8969c78f5414012ccd80a149e", + "sha256:669795516d62f38845c7033679c648903200980d68935baaa17ac5c7ae03ae0c", + "sha256:75fcd60d682db3e1f8fbe2b8b0c6761937ad56d01c1dc73edf4ef2748d5b6bc4", + "sha256:9395b0a41e8b7e9a284e3be7060db9d14ad80273841c952c83a5afc241d2bd98", + "sha256:9e37c35fc4e9410093b04a77d11a34c64bf658565e30df7cbe882056088a91c1", + "sha256:a0678793096205a4d784bd99f32803ba8100f639cf3b932dc63b21621390ea7e", + "sha256:b46554ad4dafb2927f88de5a1d207398c5385edbb5c84d30b3ef187c4a3894d8", + "sha256:c867eeccd934920a800f65c6068acdd6b87e80d45cd8c8beefff783b23cdc462", + "sha256:dd0667f5be56fb1b570154c2c0516a528e02d50da121bbbb2cbb0b6f87f59bc2", + "sha256:de2b1c20494bdf47f0160bd88ed05f5e48ae5dc336b8de7cfade71abcc95c0b9", + "sha256:f1df7b2b7740dd777571c732f98adb5aad5450aee32772f1b39249c8a50386f6", + "sha256:ffca69e29079f7880c5392bf675eb8b4146479d976ae1924d01cd92b04cccbcc" + ], + "version": "==1.17.3" }, "packaging": { "hashes": [ @@ -378,23 +387,27 @@ }, "pandas": { "hashes": [ - "sha256:18d91a9199d1dfaa01ad645f7540370ba630bdcef09daaf9edf45b4b1bca0232", - "sha256:3f26e5da310a0c0b83ea50da1fd397de2640b02b424aa69be7e0784228f656c9", - "sha256:4182e32f4456d2c64619e97c58571fa5ca0993d1e8c2d9ca44916185e1726e15", - "sha256:426e590e2eb0e60f765271d668a30cf38b582eaae5ec9b31229c8c3c10c5bc21", - "sha256:5eb934a8f0dc358f0e0cdf314072286bbac74e4c124b64371395e94644d5d919", - "sha256:717928808043d3ea55b9bcde636d4a52d2236c246f6df464163a66ff59980ad8", - "sha256:8145f97c5ed71827a6ec98ceaef35afed1377e2d19c4078f324d209ff253ecb5", - "sha256:8744c84c914dcc59cbbb2943b32b7664df1039d99e834e1034a3372acb89ea4d", - "sha256:c1ac1d9590d0c9314ebf01591bd40d4c03d710bfc84a3889e5263c97d7891dee", - "sha256:cb2e197b7b0687becb026b84d3c242482f20cbb29a9981e43604eb67576da9f6", - "sha256:d4001b71ad2c9b84ff18b182cea22b7b6cbf624216da3ea06fb7af28d1f93165", - "sha256:d8930772adccb2882989ab1493fa74bd87d47c8ac7417f5dd3dd834ba8c24dc9", - "sha256:dfbb0173ee2399bc4ed3caf2d236e5c0092f948aafd0a15fbe4a0e77ee61a958", - "sha256:eebfbba048f4fa8ac711b22c78516e16ff8117d05a580e7eeef6b0c2be554c18", - "sha256:f1b21bc5cf3dbea53d33615d1ead892dfdae9d7052fa8898083bec88be20dcd2" - ], - "version": "==0.25.1" + "sha256:0f484f43658a72e7d586a74978259657839b5bd31b903e963bb1b1491ab51775", + "sha256:0ffc6f9e20e77f3a7dc8baaad9c7fd25b858b084d3a2d8ce877bc3ea804e0636", + "sha256:23e0eac646419c3057f15eb96ab821964848607bf1d4ea5a82f26565986ec5e9", + "sha256:27c0603b15b5c6fa24885253bbe49a0c289381e7759385c59308ba4f0b166cf1", + "sha256:397fe360643fffc5b26b41efdf608647e3334a618d185a07976cd2dc51c90bce", + "sha256:3dbb3aa41c01504255bff2bd56944bdb915f6c9ce4bac7e2868efbace0b2a639", + "sha256:4e07c63247c59d61c6ebdbbb50196143cec6c5044403510c4e1a9d31854a83d6", + "sha256:4fa6d9235c6d2fecbeca82c3d326abd255866cafbfd37f66a0e826544e619760", + "sha256:56cb88b3876363d410a9d7724f43641ff164e2c9902d3266a648213e2efd5e6d", + "sha256:7ce1be1614455f83710b9a5dc1fc602a755bdddbe4dda1d41515062923a37bbf", + "sha256:ae1c96ffdeec376895e533107e0b0f9da16225a2184fbb24a5abc866769db75e", + "sha256:b6f27c9231be8a23de846f2302373991467dd8e397a4804d2614e8c5aa8d5a90", + "sha256:c6056067f894f9355bedcd168dd740aa849908d41c0a74756f6e38f203e941b3", + "sha256:ca91a19d1f0a280874a24dca44aadce42da7f3a7edb7e9ab7c7baad8febee2be", + "sha256:cbe4985f5c82a173f8cff6b7fe92d551addf99fb4ea9ff4eb4b1fe606bb098ec", + "sha256:e3e9893bfe80a8b8e6d56d36ebb7afe1df77db7b4068a6e2ef3636a91f6f1caa", + "sha256:e7b218e8711910dac3fed0d19376cd1ef0e386be5175965d332fd0c65d02a43b", + "sha256:ec48d18b8b63a5dbb838e8ea7892ee1034299e03f852bd9b6dffe870310414dd", + "sha256:f4ab6280277e3208a59bfa9f2e51240304d09e69ffb65abfb4a21d678b495f74" + ], + "version": "==0.25.2" }, "param": { "hashes": [ @@ -405,34 +418,38 @@ }, "pillow": { "hashes": [ - "sha256:00fdeb23820f30e43bba78eb9abb00b7a937a655de7760b2e09101d63708b64e", - "sha256:01f948e8220c85eae1aa1a7f8edddcec193918f933fb07aaebe0bfbbcffefbf1", - "sha256:08abf39948d4b5017a137be58f1a52b7101700431f0777bec3d897c3949f74e6", - "sha256:099a61618b145ecb50c6f279666bbc398e189b8bc97544ae32b8fcb49ad6b830", - "sha256:2c1c61546e73de62747e65807d2cc4980c395d4c5600ecb1f47a650c6fa78c79", - "sha256:2ed9c4f694861642401f27dc3cb99772be67cd190e84845c749dae0a06c3bfae", - "sha256:338581b30b908e111be578f0297255f6b57a51358cd16fa0e6f664c9a1f88bff", - "sha256:38c7d48a21cd06fdeee93987147b9b1c55b73b4cfcbf83240568bfbd5adee447", - "sha256:43fd026f613c8e48a25eba1a92f4d2ad7f3903c95d8c33a11611a7717d2ab654", - "sha256:4548236844327a718ce3bb182ab32a16fa2050c61e334e959f554cac052fb0df", - "sha256:5090857876c58885cfa388dc649e5db30aae98a068c26f3fd0ac9d7d9a4d9572", - "sha256:5bbba34f97a26a93f5e8dec469ca4ddd712451418add43da946dbaed7f7a98d2", - "sha256:65a28969a025a0eb4594637b6103201dc4ed2a9508bdab56ac33e43e3081c404", - "sha256:892bb52b70bd5ea9dbbc3ac44f38e84f5a04e9d8b1bff48159d96cb795b81159", - "sha256:8a9becd5cbd5062f973bcd2e7bc79483af310222de112b6541f8af1f93a3cc42", - "sha256:972a7aaeb7c4a2795b52eef52ee991ef040b31009f36deca6207a986607b55f3", - "sha256:97b119c436bfa96a92ac2ca525f7025836d4d4e64b1c9f9eff8dbaf3ff1d86f3", - "sha256:9ba37698e242223f8053cc158f130aee046a96feacbeab65893dbe94f5530118", - "sha256:b1b0e1f626a0f079c0d3696db70132fb1f29aa87c66aecb6501a9b8be64ce9f7", - "sha256:c14c1224fd1a5be2733530d648a316974dbbb3c946913562c6005a76f21ca042", - "sha256:c79a8546c48ae6465189e54e3245a97ddf21161e33ff7eaa42787353417bb2b6", - "sha256:ceb76935ac4ebdf6d7bc845482a4450b284c6ccfb281e34da51d510658ab34d8", - "sha256:e22bffaad04b4d16e1c091baed7f2733fc1ebb91e0c602abf1b6834d17158b1f", - "sha256:ec883b8e44d877bda6f94a36313a1c6063f8b1997aa091628ae2f34c7f97c8d5", - "sha256:f1baa54d50ec031d1a9beb89974108f8f2c0706f49798f4777df879df0e1adb6", - "sha256:f53a5385932cda1e2c862d89460992911a89768c65d176ff8c50cddca4d29bed" - ], - "version": "==6.2.0" + "sha256:047d9473cf68af50ac85f8ee5d5f21a60f849bc17d348da7fc85711287a75031", + "sha256:0f66dc6c8a3cc319561a633b6aa82c44107f12594643efa37210d8c924fc1c71", + "sha256:12c9169c4e8fe0a7329e8658c7e488001f6b4c8e88740e76292c2b857af2e94c", + "sha256:248cffc168896982f125f5c13e9317c059f74fffdb4152893339f3be62a01340", + "sha256:27faf0552bf8c260a5cee21a76e031acaea68babb64daf7e8f2e2540745082aa", + "sha256:285edafad9bc60d96978ed24d77cdc0b91dace88e5da8c548ba5937c425bca8b", + "sha256:384b12c9aa8ef95558abdcb50aada56d74bc7cc131dd62d28c2d0e4d3aadd573", + "sha256:38950b3a707f6cef09cd3cbb142474357ad1a985ceb44d921bdf7b4647b3e13e", + "sha256:4aad1b88933fd6dc2846552b89ad0c74ddbba2f0884e2c162aa368374bf5abab", + "sha256:4ac6148008c169603070c092e81f88738f1a0c511e07bd2bb0f9ef542d375da9", + "sha256:4deb1d2a45861ae6f0b12ea0a786a03d19d29edcc7e05775b85ec2877cb54c5e", + "sha256:59aa2c124df72cc75ed72c8d6005c442d4685691a30c55321e00ed915ad1a291", + "sha256:5a47d2123a9ec86660fe0e8d0ebf0aa6bc6a17edc63f338b73ea20ba11713f12", + "sha256:5cc901c2ab9409b4b7ac7b5bcc3e86ac14548627062463da0af3b6b7c555a871", + "sha256:6c1db03e8dff7b9f955a0fb9907eb9ca5da75b5ce056c0c93d33100a35050281", + "sha256:7ce80c0a65a6ea90ef9c1f63c8593fcd2929448613fc8da0adf3e6bfad669d08", + "sha256:809c19241c14433c5d6135e1b6c72da4e3b56d5c865ad5736ab99af8896b8f41", + "sha256:83792cb4e0b5af480588601467c0764242b9a483caea71ef12d22a0d0d6bdce2", + "sha256:846fa202bd7ee0f6215c897a1d33238ef071b50766339186687bd9b7a6d26ac5", + "sha256:9f5529fc02009f96ba95bea48870173426879dc19eec49ca8e08cd63ecd82ddb", + "sha256:a423c2ea001c6265ed28700df056f75e26215fd28c001e93ef4380b0f05f9547", + "sha256:ac4428094b42907aba5879c7c000d01c8278d451a3b7cccd2103e21f6397ea75", + "sha256:b1ae48d87f10d1384e5beecd169c77502fcc04a2c00a4c02b85f0a94b419e5f9", + "sha256:bf4e972a88f8841d8fdc6db1a75e0f8d763e66e3754b03006cbc3854d89f1cb1", + "sha256:c6414f6aad598364aaf81068cabb077894eb88fed99c6a65e6e8217bab62ae7a", + "sha256:c710fcb7ee32f67baf25aa9ffede4795fd5d93b163ce95fdc724383e38c9df96", + "sha256:c7be4b8a09852291c3c48d3c25d1b876d2494a0a674980089ac9d5e0d78bd132", + "sha256:c9e5ffb910b14f090ac9c38599063e354887a5f6d7e6d26795e916b4514f2c1a", + "sha256:e0697b826da6c2472bb6488db4c0a7fa8af0d52fa08833ceb3681358914b14e5", + "sha256:e9a3edd5f714229d41057d56ac0f39ad9bdba6767e8c888c951869f0bdd129b0" + ], + "version": "==6.2.1" }, "pkginfo": { "hashes": [ @@ -478,11 +495,11 @@ }, "pylint": { "hashes": [ - "sha256:7edbae11476c2182708063ac387a8f97c760d9cfe36a5ede0ca996f90cf346c8", - "sha256:844ce067788028c1a35086a5c66bc5e599ddd851841c41d6ee1623b36774d9f2" + "sha256:7b76045426c650d2b0f02fc47c14d7934d17898779da95288a74c2a7ec440702", + "sha256:856476331f3e26598017290fd65bebe81c960e806776f324093a46b76fb2d1c0" ], "index": "pypi", - "version": "==2.4.2" + "version": "==2.4.3" }, "pyparsing": { "hashes": [ @@ -616,20 +633,25 @@ }, "typed-ast": { "hashes": [ + "sha256:1170afa46a3799e18b4c977777ce137bb53c7485379d9706af8a59f2ea1aa161", "sha256:18511a0b3e7922276346bcb47e2ef9f38fb90fd31cb9223eed42c85d1312344e", "sha256:262c247a82d005e43b5b7f69aff746370538e176131c32dda9cb0f324d27141e", "sha256:2b907eb046d049bcd9892e3076c7a6456c93a25bebfe554e931620c90e6a25b0", "sha256:354c16e5babd09f5cb0ee000d54cfa38401d8b8891eefa878ac772f827181a3c", + "sha256:48e5b1e71f25cfdef98b013263a88d7145879fbb2d5185f2a0c79fa7ebbeae47", "sha256:4e0b70c6fc4d010f8107726af5fd37921b666f5b31d9331f0bd24ad9a088e631", "sha256:630968c5cdee51a11c05a30453f8cd65e0cc1d2ad0d9192819df9978984529f4", "sha256:66480f95b8167c9c5c5c87f32cf437d585937970f3fc24386f313a4c97b44e34", "sha256:71211d26ffd12d63a83e079ff258ac9d56a1376a25bc80b1cdcdf601b855b90b", + "sha256:7954560051331d003b4e2b3eb822d9dd2e376fa4f6d98fee32f452f52dd6ebb2", + "sha256:838997f4310012cf2e1ad3803bce2f3402e9ffb71ded61b5ee22617b3a7f6b6e", "sha256:95bd11af7eafc16e829af2d3df510cecfd4387f6453355188342c3e79a2ec87a", "sha256:bc6c7d3fa1325a0c6613512a093bc2a2a15aeec350451cbdf9e1d4bffe3e3233", "sha256:cc34a6f5b426748a507dd5d1de4c1978f2eb5626d51326e43280941206c209e1", "sha256:d755f03c1e4a51e9b24d899561fec4ccaf51f210d52abdf8c07ee2849b212a36", "sha256:d7c45933b1bdfaf9f36c579671fec15d25b06c8398f113dab64c18ed1adda01d", "sha256:d896919306dd0aa22d0132f62a1b78d11aaf4c9fc5b3410d3c666b818191630a", + "sha256:fdc1c9bbf79510b76408840e009ed65958feba92a88833cdceecff93ae8fff66", "sha256:ffde2fbfad571af120fcbfbbc61c72469e72f550d676c3342492a9dfdefb8f12" ], "markers": "implementation_name == 'cpython' and python_version < '3.8'", diff --git a/README.md b/README.md index 13a2fd33..72ea3c3e 100644 --- a/README.md +++ b/README.md @@ -77,6 +77,7 @@ $ docker run -it -e ANNOFAB_USER_ID=XXXX -e ANNOFAB_PASSWORD=YYYYY annofab-cli a |inspection_comment| list_unprocessed | 未処置の検査コメントを出力します。 |-| |instruction| upload | HTMLファイルを作業ガイドとして登録します。 |チェッカー/オーナ| |job|list | ジョブ一覧を出力します。 |-| +|job|list_last | 複数のプロジェクトに対して、最新のジョブを出力します。 |-| |organization_member|list | 組織メンバ一覧を出力します。 |-| |project| copy | プロジェクトをコピーします。 |オーナ and 組織管理者/組織オーナ| |project| diff | プロジェクト間の差分を表示します。 |チェッカー/オーナ| @@ -622,6 +623,25 @@ $ annofabcli job list --project_id prj1 --job_type gen-tasks --job_query '{"limi +### job list_last +複数のプロジェクトに対して、最新のジョブを出力します。 + +``` +# prj1, prj2に対して、「アノテーション更新」のジョブを出力します。 +$ annofabcli job list_last --project_id prj1 prj2 --job_type gen-annotation + +# 組織 org1配下のプロジェクト（進行中で、自分自身が所属している）に対して、「タスク全件ファイル更新」のジョブを出力します。 +$ annofabcli job list_last --organization org1 --job_type gen-tasks-list + +# アノテーションの最終更新日時を、タスクの最終更新日時と比較して出力します。 +$ annofabcli job list_last --project_id prj1 --job_type gen-annotation --add_details \ + --csv_format '{"columns": ["project_id","project_title","job_status","updated_datetime", "task_last_updated_datetime"]}' + +``` + + + + ### organization_member list 組織メンバ一覧を出力します。 @@ -865,8 +885,13 @@ $ annofabcli statistics visualize --project_id prj1 --output_dir /tmp/output $ annofabcli statistics visualize --project_id prj1 --output_dir /tmp/output \ --task_query '{"status": "complete"}' -# 作業ディレクトリ（`.annofab-cli`）内のファイルから、統計情報を可視化する。 +# アノテーションzipを更新してから、アノテーションzipをダウンロードします。 +$ annofabcli statistics visualize --project_id prj1 --output_dir /tmp/output --update_annotation + + +# WebAPIを実行せずに、作業ディレクトリ（`.annofab-cli`）内のファイルを参照して、統計情報を可視化する。 $ annofabcli statistics visualize --project_id prj1 --not_update + ``` diff --git a/annofabcli/__version__.py b/annofabcli/__version__.py index da77e85c..666b2f71 100644 --- a/annofabcli/__version__.py +++ b/annofabcli/__version__.py @@ -1 +1 @@ -__version__ = '1.11.0' +__version__ = '1.12.0' diff --git a/annofabcli/common/cli.py b/annofabcli/common/cli.py index d2dfb7ba..4bfc98bc 100644 --- a/annofabcli/common/cli.py +++ b/annofabcli/common/cli.py @@ -94,7 +94,6 @@ def get_list_from_args(str_list: Optional[List[str]] = None) -> List[str]: 文字列のListのサイズが1で、プレフィックスが`file://`ならば、ファイルパスとしてファイルを読み込み、行をListとして返す。 そうでなければ、引数の値をそのままかえす。 ただしNoneの場合は空Listを変えす - Listが1小 """ if str_list is None or len(str_list) == 0: return [] diff --git a/annofabcli/common/utils.py b/annofabcli/common/utils.py index 44bcbf91..0a9931c4 100644 --- a/annofabcli/common/utils.py +++ b/annofabcli/common/utils.py @@ -226,7 +226,7 @@ def isoduration_to_hour(duration): def allow_404_error(function): """ - Not Found Errorを無視(許容)して、処理する。 + Not Found Error(404)を無視(許容)して、処理する。Not Foundのとき戻りはNoneになる。 リソースの存在確認などに利用する。 try-exceptを行う。また404 Errorが発生したときのエラーログを無効化する """ diff --git a/annofabcli/job/list_job.py b/annofabcli/job/list_job.py index c93dff13..5c50e913 100644 --- a/annofabcli/job/list_job.py +++ b/annofabcli/job/list_job.py @@ -70,7 +70,7 @@ def parse_args(parser: argparse.ArgumentParser): job_choices = [e.value for e in JobType] argument_parser.add_project_id() - parser.add_argument('--job_type', type=str, choices=job_choices, help='ジョブタイプを指定します。') + parser.add_argument('--job_type', type=str, choices=job_choices, required=True, help='ジョブタイプを指定します。') # クエリがうまく動かないので、コメントアウトする # parser.add_argument( diff --git a/annofabcli/job/list_last_job.py b/annofabcli/job/list_last_job.py new file mode 100644 index 00000000..ba0c4853 --- /dev/null +++ b/annofabcli/job/list_last_job.py @@ -0,0 +1,169 @@ +import argparse +import logging +import sys +from typing import Any, Callable, Dict, List, Optional, Tuple, Union # pylint: disable=unused-import + +from annofabapi.models import JobInfo, JobType, Project + +import annofabcli +from annofabcli import AnnofabApiFacade +from annofabcli.common.cli import AbstractCommandLineInterface, ArgumentParser, build_annofabapi_resource_and_login +from annofabcli.common.enums import FormatArgument + +logger = logging.getLogger(__name__) + + +class ListLastJob(AbstractCommandLineInterface): + """ + ジョブ一覧を表示する。 + """ + def get_last_job(self, project_id: str, job_type: JobType) -> Optional[JobInfo]: + """ + 最新のジョブを取得する。ジョブが存在しない場合はNoneを返す。 + + """ + query_params = {"type": job_type.value} + content, _ = self.service.api.get_project_job(project_id, query_params) + job_list = content["list"] + if len(job_list) == 0: + logger.info(f"project_id={project_id}にjob_type={job_type.value}のジョブは存在しませんでした。") + return None + else: + return job_list[-1] + + def add_properties_to_job(self, job: JobInfo, project_id: str, add_details: bool = False, + project: Optional[Project] = None) -> JobInfo: + """ + ジョブ情報にプロパティを追加する。 + + Args: + project_id: + job: + add_details: + + Returns: + + """ + if project is None: + project, _ = self.service.api.get_project(project_id) + + job["project_title"] = project["title"] + + if add_details: + job["task_last_updated_datetime"] = project["summary"]["last_tasks_updated_datetime"] + + annotation_specs_history = self.service.api.get_annotation_specs_histories(project_id)[0] + job["annotation_specs_last_updated_datetime"] = annotation_specs_history[-1]["updated_datetime"] + + return job + + @annofabcli.utils.allow_404_error + def get_project(self, project_id: str) -> Project: + project, _ = self.service.api.get_project(project_id) + return project + + def get_last_job_list(self, project_id_list: List[str], job_type: JobType, + add_details: bool = False) -> List[JobInfo]: + job_list = [] + + for project_id in project_id_list: + project = self.get_project(project_id) + + if project is None: + logger.warning(f"project_id='{project_id}' のプロジェクトは存在しませんでした。") + continue + + job = self.get_last_job(project_id, job_type) + if job is None: + job = {"project_id": project_id, "project_title": project["title"]} + else: + job = self.add_properties_to_job(job, project_id=project_id, add_details=add_details, project=project) + + job_list.append(job) + + return job_list + + def print_job_list(self, project_id_list: List[str], job_type: JobType, add_details: bool = False) -> None: + """ + ジョブ一覧を出力する + + Args: + project_id: 対象のproject_id + job_type: ジョブタイプ + """ + + job_list = self.get_last_job_list(project_id_list, job_type=job_type, add_details=add_details) + logger.info(f"{len(job_list)} 個のプロジェクトの, job_type={job_type.value} の最新ジョブを出力します。") + self.print_according_to_format(job_list) + + def get_project_id_list(self, organization_name: str) -> List[str]: + """ + 進行中で自分自身が所属する project_id を取得する + + Args: + organization_name: + + Returns: + + """ + my_account, _ = self.service.api.get_my_account() + query_params = {"status": "active", "account_id": my_account["account_id"]} + project_list = self.service.wrapper.get_all_projects_of_organization(organization_name, + query_params=query_params) + return [e["project_id"] for e in project_list] + + def main(self): + args = self.args + job_type = JobType(args.job_type) + + if args.organization is not None: + project_id_list = self.get_project_id_list(args.organization) + + elif args.project_id is not None: + project_id_list = annofabcli.common.cli.get_list_from_args(args.project_id) + + else: + print("引数に`--project_id` または `--organization` を指定してください。", file=sys.stderr) + return + + self.print_job_list(project_id_list, job_type=job_type, add_details=args.add_details) + + +def main(args): + service = build_annofabapi_resource_and_login() + facade = AnnofabApiFacade(service) + ListLastJob(service, facade, args).main() + + +def parse_args(parser: argparse.ArgumentParser): + argument_parser = ArgumentParser(parser) + + job_choices = [e.value for e in JobType] + parser.add_argument('--job_type', type=str, choices=job_choices, required=True, help='ジョブタイプを指定します。') + + list_group = parser.add_mutually_exclusive_group(required=True) + list_group.add_argument('-p', '--project_id', type=str, nargs='+', + help='対象のプロジェクトのproject_idを指定してください。`file://`を先頭に付けると、一覧が記載されたファイルを指定できます。') + + list_group.add_argument('-org', '--organization', type=str, help='組織配下のすべてのプロジェクトのジョブを出力したい場合は、組織名を指定してください。') + + parser.add_argument( + '--add_details', action='store_true', help='プロジェクトに関する詳細情報を表示します' + '（`task_last_updated_datetime, annotation_specs_last_updated_datetime`）') + + argument_parser.add_format(choices=[FormatArgument.CSV, FormatArgument.JSON, FormatArgument.PRETTY_JSON], + default=FormatArgument.CSV) + argument_parser.add_output() + argument_parser.add_csv_format() + + argument_parser.add_query() + parser.set_defaults(subcommand_func=main) + + +def add_parser(subparsers: argparse._SubParsersAction): + subcommand_name = "list_last" + subcommand_help = "複数のプロジェクトに対して、最新のジョブを出力します。" + description = ("複数のプロジェクトに対して、最新のジョブを出力します。") + + parser = annofabcli.common.cli.add_parser(subparsers, subcommand_name, subcommand_help, description) + parse_args(parser) diff --git a/annofabcli/job/subcommand_job.py b/annofabcli/job/subcommand_job.py index efa963be..0f2eb71e 100644 --- a/annofabcli/job/subcommand_job.py +++ b/annofabcli/job/subcommand_job.py @@ -3,6 +3,7 @@ import annofabcli import annofabcli.common.cli import annofabcli.job.list_job +import annofabcli.job.list_last_job def parse_args(parser: argparse.ArgumentParser): @@ -11,6 +12,7 @@ def parse_args(parser: argparse.ArgumentParser): # サブコマンドの定義 annofabcli.job.list_job.add_parser(subparsers) + annofabcli.job.list_last_job.add_parser(subparsers) def add_parser(subparsers: argparse._SubParsersAction): diff --git a/annofabcli/project/download.py b/annofabcli/project/download.py index 59637ce8..73ce873f 100644 --- a/annofabcli/project/download.py +++ b/annofabcli/project/download.py @@ -30,7 +30,8 @@ def download(self, target: DownloadTarget, project_id: str, output: str, latest: if latest: self.service.api.post_project_tasks_update(project_id) result = self.service.wrapper.wait_for_completion(project_id, job_type=JobType.GEN_TASKS_LIST, - job_access_interval=60, max_job_access=30) + job_access_interval=JOB_ACCESS_INTERVAL, + max_job_access=MAX_JOB_ACCESS) if result: logger.info(f"タスクファイルの更新が完了しました。") else: diff --git a/annofabcli/statistics/database.py b/annofabcli/statistics/database.py index 4936370a..3e686155 100644 --- a/annofabcli/statistics/database.py +++ b/annofabcli/statistics/database.py @@ -4,19 +4,24 @@ import os import pickle import shutil +import zipfile from pathlib import Path -from typing import Any, Dict, List, Optional, Set +from typing import Any, Callable, Dict, List, Optional, Set import annofabapi import annofabapi.utils +import dateutil import more_itertools -from annofabapi.dataclass.annotation import FullAnnotationDetail -from annofabapi.models import Annotation, InputDataId, Inspection, Task, TaskHistory, TaskId -from annofabapi.parser import lazy_parse_full_annotation_zip +from annofabapi.dataclass.annotation import SimpleAnnotationDetail +from annofabapi.models import InputDataId, Inspection, JobStatus, JobType, Task, TaskHistory, TaskId +from annofabapi.parser import SimpleAnnotationZipParser logger = logging.getLogger(__name__) -AnnotationDict = Dict[TaskId, Dict[InputDataId, List[FullAnnotationDetail]]] +InputDataDict = Dict[InputDataId, Dict[str, Any]] +AnnotationDict = Dict[TaskId, InputDataDict] +AnnotationSummaryFunc = Callable[[List[SimpleAnnotationDetail]], Dict[str, Any]] +"""アノテーションの概要を算出する関数""" class Database: @@ -50,8 +55,7 @@ def __init__(self, annofab_service: annofabapi.Resource, project_id: str, chekpo # ダウンロードした一括情報 self.tasks_json_path = Path(f"{self.checkpoint_dir}/tasks.json") self.inspection_json_path = Path(f"{self.checkpoint_dir}/inspections.json") - self.annotations_dir_path = Path(f"{self.checkpoint_dir}/full-annotations") - self.annotations_zip_path = Path(f"{self.checkpoint_dir}/full-annotations.zip") + self.annotations_zip_path = Path(f"{self.checkpoint_dir}/simple-annotations.zip") def __write_checkpoint(self, obj, pickle_file_name): """ @@ -98,14 +102,6 @@ def read_account_statistics_from_checkpoint(self) -> List[Dict[str, Any]]: result = self.__read_checkpoint("account_statistics.pickel") return result if result is not None else [] - @staticmethod - def _read_annotations_from_json(input_data_json: Path) -> List[Annotation]: - with open(str(input_data_json)) as f: - full_annotation = json.load(f) - - annotation_list = full_annotation["detail"] - return annotation_list - @staticmethod def task_exists(task_list: List[Task], task_id) -> bool: task = more_itertools.first_true(task_list, pred=lambda e: e["task_id"] == task_id) @@ -114,27 +110,34 @@ def task_exists(task_list: List[Task], task_id) -> bool: else: return True - def read_annotations_from_full_annotion_dir(self, task_list: List[Task]) -> AnnotationDict: - logger.debug(f"reading {self.annotations_dir_path}") + def read_annotation_summary(self, task_list: List[Task], + annotation_summary_func: AnnotationSummaryFunc) -> AnnotationDict: + logger.info(f"reading {str(self.annotations_zip_path)}") - tasks_dict: AnnotationDict = {} - iter_parser = lazy_parse_full_annotation_zip(self.annotations_zip_path) - for parser in iter_parser: - full_annotation = parser.parse() - task_id = full_annotation.task_id - if not self.task_exists(task_list, task_id): - continue + def read_annotation_summary(task_id: str, input_data_id_: str) -> Dict[str, Any]: + json_path = f"{task_id}/{input_data_id_}.json" + parser = SimpleAnnotationZipParser(zip_file, json_path) + simple_annotation = parser.parse() + return annotation_summary_func(simple_annotation.details) - input_data_id = full_annotation.input_data_id + with zipfile.ZipFile(self.annotations_zip_path, 'r') as zip_file: + annotation_dict: AnnotationDict = {} - input_data_dict = tasks_dict.get(task_id) - if input_data_dict is None: - input_data_dict = {} + task_count = 0 + for task in task_list: + task_count += 1 + if task_count % 1000 == 0: + logger.debug(f"{task_count} / {len(task_list)} 件目を読み込み中") - input_data_dict[input_data_id] = full_annotation.detail - tasks_dict[task_id] = input_data_dict + task_id = task["task_id"] + input_data_id_list = task["input_data_id_list"] - return tasks_dict + input_data_dict: InputDataDict = {} + for input_data_id in input_data_id_list: + input_data_dict[input_data_id] = read_annotation_summary(task_id, input_data_id) + + annotation_dict[task_id] = input_data_dict + return annotation_dict def read_inspections_from_json(self, task_id_list: List[TaskId]) -> Dict[TaskId, Dict[InputDataId, List[Inspection]]]: @@ -163,13 +166,79 @@ def read_inspections_from_json(self, return tasks_dict - def _download_full_annotation(self, dest_path: str): - _, response = self.annofab_service.api.get_archive_full_with_pro_id(self.project_id) - url = response.headers['Location'] - annofabapi.utils.download(url, dest_path) + def wait_for_completion_updated_annotation(self, project_id): + MAX_JOB_ACCESS = 120 + JOB_ACCESS_INTERVAL = 60 + MAX_WAIT_MINUTU = MAX_JOB_ACCESS * JOB_ACCESS_INTERVAL / 60 + result = self.annofab_service.wrapper.wait_for_completion(project_id, job_type=JobType.GEN_ANNOTATION, + job_access_interval=JOB_ACCESS_INTERVAL, + max_job_access=MAX_JOB_ACCESS) + if result: + logger.info(f"アノテーションの更新が完了しました。") + else: + logger.info(f"アノテーションの更新に失敗しました or {MAX_WAIT_MINUTU} 分待っても、更新が完了しませんでした。") + return + + def update_annotation_zip(self, project_id: str, should_update_annotation_zip: bool = False): + """ + 必要に応じて、アノテーションの更新を実施 and アノテーションの更新を待つ。 - def _download_db_file(self): - """DBになりうるファイルをダウンロードする""" + Args: + project_id: + should_update_annotation_zip: + + Returns: + + """ + + project, _ = self.annofab_service.api.get_project(project_id) + last_tasks_updated_datetime = project['summary']['last_tasks_updated_datetime'] + logger.debug(f"タスクの最終更新日時={last_tasks_updated_datetime}") + + annotation_specs_history = self.annofab_service.api.get_annotation_specs_histories(project_id)[0] + annotation_specs_updated_datetime = annotation_specs_history[-1]["updated_datetime"] + logger.debug(f"アノテーション仕様の最終更新日時={annotation_specs_updated_datetime}") + + job = self.annofab_service.api.get_project_job(project_id, query_params={ + "type": "gen-annotation", + "limit": 1 + })[0]["list"][0] + logger.debug(f"アノテーションzipの最終更新日時={job['updated_datetime']}, job_status={job['job_status']}") + + if should_update_annotation_zip: + job_status = JobStatus(job['job_status']) + if job_status == JobStatus.PROGRESS: + logger.info(f"アノテーション更新が完了するまで待ちます。") + self.wait_for_completion_updated_annotation(project_id) + + elif job_status == JobStatus.SUCCEEDED: + if dateutil.parser.parse(job['updated_datetime']) < dateutil.parser.parse(last_tasks_updated_datetime): + logger.info(f"タスクの最新更新日時よりアノテーションzipの最終更新日時の方が古いので、アノテーションzipを更新します。") + self.annofab_service.api.post_annotation_archive_update(project_id) + self.wait_for_completion_updated_annotation(project_id) + + elif dateutil.parser.parse( + job['updated_datetime']) < dateutil.parser.parse(annotation_specs_updated_datetime): + logger.info(f"アノテーション仕様の更新日時よりアノテーションzipの最終更新日時の方が古いので、アノテーションzipを更新します。") + self.annofab_service.api.post_annotation_archive_update(project_id) + self.wait_for_completion_updated_annotation(project_id) + + else: + logger.info(f"アノテーションzipを更新する必要がないので、更新しません。") + + elif job_status == JobStatus.FAILED: + logger.info("アノテーションzipの更新に失敗しているので、アノテーションzipを更新します。") + self.annofab_service.api.post_annotation_archive_update(project_id) + self.wait_for_completion_updated_annotation(project_id) + + def _download_db_file(self, should_update_annotation_zip: bool = False): + """ + DBになりうるファイルをダウンロードする + + Args: + update_annotation: Trunならアノテーションzipを更新する。ただしタスクの最終更新日時が、今のアノテーションzipの最終更新日時より + + """ logger.debug(f"downloading {str(self.tasks_json_path)}") self.annofab_service.wrapper.download_project_tasks_url(self.project_id, str(self.tasks_json_path)) @@ -177,15 +246,13 @@ def _download_db_file(self): logger.debug(f"downloading {str(self.inspection_json_path)}") self.annofab_service.wrapper.download_project_inspections_url(self.project_id, str(self.inspection_json_path)) - annotations_zip_file = f"{self.checkpoint_dir}/full-annotations.zip" + annotations_zip_file = f"{self.checkpoint_dir}/simple-annotations.zip" + + self.update_annotation_zip(self.project_id, should_update_annotation_zip) logger.debug(f"downloading {str(annotations_zip_file)}") - self._download_full_annotation(annotations_zip_file) + self.annofab_service.wrapper.download_annotation_archive(self.project_id, annotations_zip_file, v2=True) # task historiesは未完成なので、使わない - # 前のディレクトリを削除する - if self.annotations_dir_path.exists(): - shutil.rmtree(str(self.annotations_dir_path)) - def read_tasks_from_json(self, task_query_param: Optional[Dict[str, Any]] = None, ignored_task_id_list: Optional[List[TaskId]] = None) -> List[Task]: """ @@ -219,20 +286,22 @@ def filter_task(arg_task): return [task for task in all_tasks if filter_task(task)] - def update_db(self, task_query_param: Dict[str, Any], ignored_task_ids: Optional[List[str]] = None): + def update_db(self, task_query_param: Dict[str, Any], ignored_task_ids: Optional[List[str]] = None, + should_update_annotation_zip: bool = False): """ Annofabから情報を取得し、DB（pickelファイル）を更新する。 TODO タスク履歴一括ファイルに必要な情報が含まれたら、修正する Args: task_query_param: タスク取得のクエリパラメタ. デフォルトは受け入れ完了タスク ignored_task_ids: 可視化対象外のtask_idのList + should_update_annotation_zip: アノテーションzipを更新するかどうか """ # 残すべきファイル self.filename_timestamp = "{0:%Y%m%d-%H%M%S}".format(datetime.datetime.now()) # DB用のJSONファイルをダウンロードする - self._download_db_file() + self._download_db_file(should_update_annotation_zip) logger.info(f"DB更新: task_query_param = {task_query_param}") tasks = self.read_tasks_from_json(task_query_param) @@ -265,14 +334,14 @@ def get_not_updated_task_ids(old_tasks, new_tasks) -> Set[str]: Returns: """ - from dateutil.parser import parse updated_task_ids = set() for new_task in new_tasks: filterd_list = [e for e in old_tasks if e["task_id"] == new_task["task_id"]] if len(filterd_list) == 0: continue old_task = filterd_list[0] - if parse(old_task["updated_datetime"]) == parse(new_task["updated_datetime"]): + if dateutil.parser.parse(old_task["updated_datetime"]) == dateutil.parser.parse( + new_task["updated_datetime"]): updated_task_ids.add(new_task["task_id"]) return updated_task_ids diff --git a/annofabcli/statistics/graph.py b/annofabcli/statistics/graph.py index 4ef6e30a..7372dfcd 100644 --- a/annofabcli/statistics/graph.py +++ b/annofabcli/statistics/graph.py @@ -14,6 +14,8 @@ logger = logging.getLogger(__name__) +hv.extension('bokeh') + class Graph: """ diff --git a/annofabcli/statistics/table.py b/annofabcli/statistics/table.py index 8f0d6058..6751d726 100644 --- a/annofabcli/statistics/table.py +++ b/annofabcli/statistics/table.py @@ -4,7 +4,7 @@ import dateutil.parser import pandas as pd -from annofabapi.dataclass.annotation import FullAnnotationDetail +from annofabapi.dataclass.annotation import SimpleAnnotationDetail from annofabapi.models import InputDataId, Inspection, Task, TaskHistory, TaskId, TaskPhase import annofabcli @@ -32,7 +32,7 @@ class Table: _task_id_list: Optional[List[TaskId]] = None _task_list: Optional[List[Task]] = None _inspections_dict: Optional[Dict[TaskId, Dict[InputDataId, List[Inspection]]]] = None - _annotations_dict: Optional[Dict[TaskId, Dict[InputDataId, List[FullAnnotationDetail]]]] = None + _annotations_dict: Optional[Dict[TaskId, Dict[InputDataId, Dict[str, Any]]]] = None def __init__(self, database: Database, task_query_param: Dict[str, Any], ignored_task_id_list: Optional[List[TaskId]] = None): @@ -69,9 +69,22 @@ def _get_annotations_dict(self) -> AnnotationDict: return self._annotations_dict else: task_list = self._get_task_list() - self._annotations_dict = self.database.read_annotations_from_full_annotion_dir(task_list) + self._annotations_dict = self.database.read_annotation_summary(task_list, self._create_annotation_summary) return self._annotations_dict + def _create_annotation_summary(self, annotation_list: List[SimpleAnnotationDetail]) -> Dict[str, Any]: + annotation_summary = {} + annotation_summary["total_count"] = len(annotation_list) + + # labelごとのアノテーション数を算出 + for label_name in self.label_dict.values(): + annotation_count = 0 + key = f"label_{label_name}" + annotation_count += len([e for e in annotation_list if e.label == label_name]) + annotation_summary[key] = annotation_count + + return annotation_summary + @staticmethod def _inspection_condition(inspection_arg, exclude_reply: bool, only_error_corrected: bool): """ @@ -127,27 +140,9 @@ def _update_annotaion_specs(self): self.label_dict = self.get_labels_dict(annotaion_specs["labels"]) logger.debug("annofab_service.wrapper.get_all_project_members()") - self.project_members_dict = self.get_project_members_dict() - - @staticmethod - def _get_acceptance_count_and_histories(task_histories: List[TaskHistory]) -> List[Tuple[int, TaskHistory]]: - """ - 受入で差し戻しごとにグループ分けする。 - """ - - new_list: List[Tuple[int, TaskHistory]] = [] - rejections_by_phase = 0 - for i, history in enumerate(task_histories): - if history['phase'] == 'annotation': - if i - 1 >= 0 and task_histories[i - 1]['phase'] == 'acceptance': - rejections_by_phase += 1 + self.project_members_dict = self._get_project_members_dict() - elm = rejections_by_phase, history - new_list.append(elm) - - return new_list - - def get_project_members_dict(self) -> Dict[str, Any]: + def _get_project_members_dict(self) -> Dict[str, Any]: project_members_dict = {} project_members = self.annofab_service.wrapper.get_all_project_members(self.project_id) @@ -160,7 +155,7 @@ def get_labels_dict(labels) -> Dict[str, str]: """ ラベル情報を設定する Returns: - key: label_id, value: label_name(ja) + key: label_id, value: label_name(en) """ label_dict = {} @@ -168,7 +163,7 @@ def get_labels_dict(labels) -> Dict[str, str]: for e in labels: label_id = e["label_id"] messages_list = e["label_name"]["messages"] - label_name = [m["message"] for m in messages_list if m["lang"] == "ja-JP"][0] + label_name = [m["message"] for m in messages_list if m["lang"] == "en-US"][0] label_dict[label_id] = label_name @@ -318,10 +313,13 @@ def create_task_df(self) -> pd.DataFrame: annotation_worktime_hour, inspection_worktime_hour, acceptance_worktime_hour, sum_worktime_hour """ def set_annotation_info(arg_task): - input_data_dict = annotations_dict[arg_task["task_id"]] total_annotation_count = 0 - for annotation_list in input_data_dict.values(): - total_annotation_count += len(annotation_list) + + input_data_id_list = arg_task["input_data_id_list"] + input_data_dict = annotations_dict[arg_task["task_id"]] + + for input_data_id in input_data_id_list: + total_annotation_count += input_data_dict[input_data_id]["total_count"] arg_task["annotation_count"] = total_annotation_count @@ -343,6 +341,7 @@ def set_inspection_info(arg_task): arg_task["inspection_count"] = inspection_count arg_task["input_data_count_of_inspection"] = input_data_count_of_inspection + logger.info(f"execute `create_task_df` function") tasks = self._get_task_list() task_histories_dict = self.database.read_task_histories_from_checkpoint() inspections_dict = self._get_inspections_dict() @@ -383,18 +382,19 @@ def create_task_for_annotation_df(self): task_list = [] for task in tasks: task_id = task["task_id"] - + input_data_dict = annotations_dict[task_id] new_task = {} for key in ["task_id", "phase", "status"]: new_task[key] = task[key] new_task["input_data_count"] = len(task["input_data_id_list"]) + input_data_id_list = task["input_data_id_list"] - input_data_dict = annotations_dict[task_id] - for label_id, label_name in self.label_dict.items(): + for label_name in self.label_dict.values(): annotation_count = 0 - for annotation_list in input_data_dict.values(): - annotation_count += len([e for e in annotation_list if e.label_id == label_id]) + for input_data_id in input_data_id_list: + annotation_summary = input_data_dict[input_data_id] + annotation_count += annotation_summary[f"label_{label_name}"] new_task[label_name] = annotation_count diff --git a/annofabcli/statistics/visualize_statistics.py b/annofabcli/statistics/visualize_statistics.py index 7b552e84..dd8a3c0c 100644 --- a/annofabcli/statistics/visualize_statistics.py +++ b/annofabcli/statistics/visualize_statistics.py @@ -36,7 +36,8 @@ class VisualizeStatistics(AbstractCommandLineInterface): 統計情報を可視化する。 """ def visualize_statistics(self, project_id: str, work_dir: Path, output_dir: Path, task_query: Dict[str, Any], - ignored_task_id_list: List[TaskId], user_id_list: List[str], update: bool = False): + ignored_task_id_list: List[TaskId], user_id_list: List[str], update: bool = False, + should_update_annotation_zip: bool = False): """ タスク一覧を出力する @@ -53,7 +54,8 @@ def visualize_statistics(self, project_id: str, work_dir: Path, output_dir: Path database = Database(self.service, project_id, str(checkpoint_dir)) if update: - database.update_db(task_query, ignored_task_id_list) + database.update_db(task_query, ignored_task_id_list, + should_update_annotation_zip=should_update_annotation_zip) table_obj = Table(database, task_query, ignored_task_id_list) write_project_name_file(self.service, project_id, output_dir) @@ -61,16 +63,20 @@ def visualize_statistics(self, project_id: str, work_dir: Path, output_dir: Path graph_obj = Graph(table_obj, str(output_dir)) # TSV出力 - tsv_obj.write_task_list(dropped_columns=["histories_by_phase", "input_data_id_list"]) - tsv_obj.write_inspection_list(dropped_columns=["data"]) - tsv_obj.write_member_list() - tsv_obj.write_ラベルごとのアノテーション数() + task_df = table_obj.create_task_df() + inspection_df = table_obj.create_inspection_df() + member_df = table_obj.create_member_df(task_df) + annotation_df = table_obj.create_task_for_annotation_df() + + tsv_obj.write_task_list(arg_df=task_df, dropped_columns=["histories_by_phase", "input_data_id_list"]) + tsv_obj.write_inspection_list(arg_df=inspection_df, dropped_columns=["data"]) + tsv_obj.write_member_list(arg_df=member_df) + tsv_obj.write_ラベルごとのアノテーション数(arg_df=annotation_df) tsv_obj.write_ユーザ別日毎の作業時間() - graph_obj.wirte_ラベルごとのアノテーション数() - graph_obj.write_プロジェクト全体のヒストグラム() - - graph_obj.write_アノテーションあたり作業時間(first_annotation_user_id_list=user_id_list) + graph_obj.wirte_ラベルごとのアノテーション数(annotation_df) + graph_obj.write_プロジェクト全体のヒストグラム(task_df) + graph_obj.write_アノテーションあたり作業時間(task_df=task_df, first_annotation_user_id_list=user_id_list) def main(self): args = self.args @@ -80,7 +86,8 @@ def main(self): self.visualize_statistics(args.project_id, output_dir=Path(args.output_dir), work_dir=Path(args.work_dir), task_query=task_query, ignored_task_id_list=ignored_task_id_list, - user_id_list=user_id_list, update=not args.not_update) + user_id_list=user_id_list, update=not args.not_update, + should_update_annotation_zip=args.update_annotation) def main(args): @@ -93,7 +100,7 @@ def parse_args(parser: argparse.ArgumentParser): argument_parser = ArgumentParser(parser) argument_parser.add_project_id() - parser.add_argument('--output_dir', type=str, required=True, help='出力ディレクトリのパス') + parser.add_argument('-o', '--output_dir', type=str, required=True, help='出力ディレクトリのパス') parser.add_argument( '-u', '--user_id', nargs='+', help=("メンバごとの統計グラフに表示するユーザのuser_idを指定してください。" @@ -101,7 +108,7 @@ def parse_args(parser: argparse.ArgumentParser): "file://`を先頭に付けると、一覧が記載されたファイルを指定できます。")) parser.add_argument( - '-tq', '--task_query', type=str, help='タスクの検索クエリをJSON形式で指定します。' + '-tq', '--task_query', type=str, help='タスクの検索クエリをJSON形式で指定します。指定しない場合はすべてのタスクを取得します。' '`file://`を先頭に付けると、JSON形式のファイルを指定できます。' 'クエリのキーは、phase, statusのみです。[getTasks API](https://annofab.com/docs/api/#operation/getTasks) 参照') @@ -110,7 +117,12 @@ def parse_args(parser: argparse.ArgumentParser): "指定しない場合は、すべてのタスクが可視化対象です。" "file://`を先頭に付けると、一覧が記載されたファイルを指定できます。")) - parser.add_argument('--not_update', action="store_true", help='APIにアクセスして、作業ディレクトリ内のファイルを更新します。') + parser.add_argument('--not_update', action="store_true", help='作業ディレクトリ内のファイルを参照して、統計情報を出力します。' + 'AnnoFab Web APIへのアクセスを最小限にします。') + + parser.add_argument( + '--update_annotation', action="store_true", help='アノテーションzipを更新してから、アノテーションzipをダウンロードします。' + 'ただし、アノテーションzipの最終更新日時がタスクの最終更新日時より新しい場合は、アノテーションzipを更新しません。') parser.add_argument('--work_dir', type=str, default=".annofab-cli", help="作業ディレクトリのパス。指定しない場合カレントの'.annofab-cli'ディレクトリに保存する")