From 9321737563815e007bbf6fc5af71a8f677ba62b7 Mon Sep 17 00:00:00 2001 From: Talv Date: Wed, 24 Jul 2019 23:32:16 +0200 Subject: [PATCH 1/2] v2 init --- .clang-format | 13 + .editorconfig | 17 + .gitmodules | 6 + CMakeLists.txt | 18 +- cxxopts | 1 + include/SimpleOpt.h | 1060 -------------------- include/common.hpp | 14 + include/mplog/ColorConsoleAppenderStderr.h | 103 ++ include/mplog/ConsoleAppenderStderr.h | 64 ++ include/storage.hpp | 68 ++ include/util.hpp | 26 + plog | 1 + src/storage.cc | 86 ++ src/storm-extract.cpp | 407 -------- src/stormex.cc | 305 ++++++ src/util.cc | 93 ++ 16 files changed, 812 insertions(+), 1470 deletions(-) create mode 100644 .clang-format create mode 100644 .editorconfig create mode 160000 cxxopts delete mode 100755 include/SimpleOpt.h create mode 100644 include/common.hpp create mode 100644 include/mplog/ColorConsoleAppenderStderr.h create mode 100644 include/mplog/ConsoleAppenderStderr.h create mode 100644 include/storage.hpp create mode 100644 include/util.hpp create mode 160000 plog create mode 100644 src/storage.cc delete mode 100644 src/storm-extract.cpp create mode 100644 src/stormex.cc create mode 100644 src/util.cc diff --git a/.clang-format b/.clang-format new file mode 100644 index 0000000..f05202e --- /dev/null +++ b/.clang-format @@ -0,0 +1,13 @@ +BasedOnStyle: Webkit +BreakBeforeBraces: Custom +BraceWrapping: + BeforeElse: true + AfterControlStatement: false + # IndentWrappedFunctionNames: true + # AfterCaseLabel: true + AfterFunction: true + AfterClass: false + AfterStruct: true + AfterEnum: true + AfterUnion: true + AfterNamespace: true diff --git a/.editorconfig b/.editorconfig new file mode 100644 index 0000000..4d9b62f --- /dev/null +++ b/.editorconfig @@ -0,0 +1,17 @@ +; EditorConfig helps developers define and maintain consistent +; coding styles between different editors and IDEs. + +; For more visit http://editorconfig.org. +root = true + +; Choose between lf or rf on "end_of_line" property +[*] +indent_style = space +end_of_line = lf +charset = utf-8 +trim_trailing_whitespace = true +insert_final_newline = true +indent_size = 4 + +[*.md] +trim_trailing_whitespace = false diff --git a/.gitmodules b/.gitmodules index 729bcb8..578bfec 100644 --- a/.gitmodules +++ b/.gitmodules @@ -1,3 +1,9 @@ [submodule "CascLib"] path = CascLib url = https://github.com/ladislav-zezula/CascLib.git +[submodule "cxxopts"] + path = cxxopts + url = https://github.com/jarro2783/cxxopts.git +[submodule "plog"] + path = plog + url = https://github.com/SergiusTheBest/plog.git diff --git a/CMakeLists.txt b/CMakeLists.txt index 632f5d0..e6ff5e5 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -24,14 +24,26 @@ if(WIN32) set(CASC_BUILD_STATIC_LIB ON CACHE BOOL "Force Static library building to link test app") set(CASC_BUILD_SHARED_LIB OFF CACHE BOOL "Compile dynamically linked library") endif() + add_subdirectory(CascLib) +add_subdirectory(plog) -include_directories("${STORMEXTRACT_SOURCE_DIR}/src/" +include_directories("${STORMEXTRACT_SOURCE_DIR}/include/" "${STORMEXTRACT_SOURCE_DIR}/CascLib/src/" - "${STORMEXTRACT_SOURCE_DIR}/include/" + "${STORMEXTRACT_SOURCE_DIR}/cxxopts/include/" + "${STORMEXTRACT_SOURCE_DIR}/plog/include/" +) + +set(SRC_FILES + src/util.cc + src/storage.cc + src/stormex.cc ) -add_executable(stormex src/storm-extract.cpp) +add_executable(stormex ${SRC_FILES}) + +target_link_libraries(stormex plog) + if(WIN32) target_link_libraries(stormex casc_static) else() diff --git a/cxxopts b/cxxopts new file mode 160000 index 0000000..3c73d91 --- /dev/null +++ b/cxxopts @@ -0,0 +1 @@ +Subproject commit 3c73d91c0b04e2b59462f0a741be8c07024c1bc0 diff --git a/include/SimpleOpt.h b/include/SimpleOpt.h deleted file mode 100755 index 9e34a29..0000000 --- a/include/SimpleOpt.h +++ /dev/null @@ -1,1060 +0,0 @@ -/*! @file SimpleOpt.h - - @version 3.4 - - @brief A cross-platform command line library which can parse almost any - of the standard command line formats in use today. It is designed - explicitly to be portable to any platform and has been tested on Windows - and Linux. See CSimpleOptTempl for the class definition. - - @section features FEATURES - - - MIT Licence allows free use in all software (including GPL - and commercial) - - multi-platform (Windows 95/98/ME/NT/2K/XP, Linux, Unix) - - supports all lengths of option names: - -
- - switch character only (e.g. use stdin for input) -
-o - short (single character) -
-long - long (multiple character, single switch character) -
--longer - long (multiple character, multiple switch characters) -
- - supports all types of arguments for options: - -
--option - short/long option flag (no argument) -
--option ARG - short/long option with separate required argument -
--option=ARG - short/long option with combined required argument -
--option[=ARG] - short/long option with combined optional argument -
-oARG - short option with combined required argument -
-o[ARG] - short option with combined optional argument -
- - supports options with multiple or variable numbers of arguments: - -
--multi ARG1 ARG2 - Multiple arguments -
--multi N ARG-1 ARG-2 ... ARG-N - Variable number of arguments -
- - supports case-insensitive option matching on short, long and/or - word arguments. - - supports options which do not use a switch character. i.e. a special - word which is construed as an option. - e.g. "foo.exe open /directory/file.txt" - - supports clumping of multiple short options (no arguments) in a string - e.g. "foo.exe -abcdef file1" <==> "foo.exe -a -b -c -d -e -f file1" - - automatic recognition of a single slash as equivalent to a single - hyphen on Windows, e.g. "/f FILE" is equivalent to "-f FILE". - - file arguments can appear anywhere in the argument list: - "foo.exe file1.txt -a ARG file2.txt --flag file3.txt file4.txt" - files will be returned to the application in the same order they were - supplied on the command line - - short-circuit option matching: "--man" will match "--mandate" - invalid options can be handled while continuing to parse the command - line valid options list can be changed dynamically during command line - processing, i.e. accept different options depending on an option - supplied earlier in the command line. - - implemented with only a single C++ header file - - optionally use no C runtime or OS functions - - char, wchar_t and Windows TCHAR in the same program - - complete working examples included - - compiles cleanly at warning level 4 (Windows/VC.NET 2003), warning - level 3 (Windows/VC6) and -Wall (Linux/gcc) - - @section usage USAGE - - The SimpleOpt class is used by following these steps: - -
    -
  1. Include the SimpleOpt.h header file - - 
    -        \#include "SimpleOpt.h"
-
    - -
  2. Define an array of valid options for your program. - -
    -@link CSimpleOptTempl::SOption CSimpleOpt::SOption @endlink g_rgOptions[] = {
-    { OPT_FLAG, _T("-a"),     SO_NONE    }, // "-a"
-    { OPT_FLAG, _T("-b"),     SO_NONE    }, // "-b"
-    { OPT_ARG,  _T("-f"),     SO_REQ_SEP }, // "-f ARG"
-    { OPT_HELP, _T("-?"),     SO_NONE    }, // "-?"
-    { OPT_HELP, _T("--help"), SO_NONE    }, // "--help"
-    SO_END_OF_OPTIONS                       // END
-};
-
    - - Note that all options must start with a hyphen even if the slash will - be accepted. This is because the slash character is automatically - converted into a hyphen to test against the list of options. - For example, the following line matches both "-?" and "/?" - (on Windows). - - 
    -        { OPT_HELP, _T("-?"),     SO_NONE    }, // "-?"
-
    - -
  3. Instantiate a CSimpleOpt object supplying argc, argv and the option - table - -
    -@link CSimpleOptTempl CSimpleOpt @endlink args(argc, argv, g_rgOptions);
-
    - -
  4. Process the arguments by calling Next() until it returns false. - On each call, first check for an error by calling LastError(), then - either handle the error or process the argument. - -
    -while (args.Next()) {
-    if (args.LastError() == SO_SUCCESS) {
-        handle option: use OptionId(), OptionText() and OptionArg()
-    }
-    else {
-        handle error: see ESOError enums
-    }
-}
-
    - -
  5. Process all non-option arguments with File(), Files() and FileCount() - -
    -ShowFiles(args.FileCount(), args.Files());
-
    - -
- - @section notes NOTES - - - In MBCS mode, this library is guaranteed to work correctly only when - all option names use only ASCII characters. - - Note that if case-insensitive matching is being used then the first - matching option in the argument list will be returned. - - @section licence MIT LICENCE - - The licence text below is the boilerplate "MIT Licence" used from: - http://www.opensource.org/licenses/mit-license.php - - Copyright (c) 2006-2007, Brodie Thiesfield - - Permission is hereby granted, free of charge, to any person obtaining a - copy of this software and associated documentation files (the "Software"), - to deal in the Software without restriction, including without limitation - the rights to use, copy, modify, merge, publish, distribute, sublicense, - and/or sell copies of the Software, and to permit persons to whom the - Software is furnished to do so, subject to the following conditions: - - The above copyright notice and this permission notice shall be included - in all copies or substantial portions of the Software. - - THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS - OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF - MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. - IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY - CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, - TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE - SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. -*/ - -/*! @mainpage - - -
Library SimpleOpt -
Author Brodie Thiesfield [code at jellycan dot com] -
Source http://code.jellycan.com/simpleopt/ -
- - @section SimpleOpt SimpleOpt - - A cross-platform library providing a simple method to parse almost any of - the standard command-line formats in use today. - - See the @link SimpleOpt.h SimpleOpt @endlink documentation for full - details. - - @section SimpleGlob SimpleGlob - - A cross-platform file globbing library providing the ability to - expand wildcards in command-line arguments to a list of all matching - files. - - See the @link SimpleGlob.h SimpleGlob @endlink documentation for full - details. -*/ - -#ifndef INCLUDED_SimpleOpt -#define INCLUDED_SimpleOpt - -// Default the max arguments to a fixed value. If you want to be able to -// handle any number of arguments, then predefine this to 0 and it will -// use an internal dynamically allocated buffer instead. -#ifdef SO_MAX_ARGS -# define SO_STATICBUF SO_MAX_ARGS -#else -# include // malloc, free -# include // memcpy -# define SO_STATICBUF 50 -#endif - -//! Error values -typedef enum _ESOError -{ - //! No error - SO_SUCCESS = 0, - - /*! It looks like an option (it starts with a switch character), but - it isn't registered in the option table. */ - SO_OPT_INVALID = -1, - - /*! Multiple options matched the supplied option text. - Only returned when NOT using SO_O_EXACT. */ - SO_OPT_MULTIPLE = -2, - - /*! Option doesn't take an argument, but a combined argument was - supplied. */ - SO_ARG_INVALID = -3, - - /*! SO_REQ_CMB style-argument was supplied to a SO_REQ_SEP option - Only returned when using SO_O_PEDANTIC. */ - SO_ARG_INVALID_TYPE = -4, - - //! Required argument was not supplied - SO_ARG_MISSING = -5, - - /*! Option argument looks like another option. - Only returned when NOT using SO_O_NOERR. */ - SO_ARG_INVALID_DATA = -6 -} ESOError; - -//! Option flags -enum _ESOFlags -{ - /*! Disallow partial matching of option names */ - SO_O_EXACT = 0x0001, - - /*! Disallow use of slash as an option marker on Windows. - Un*x only ever recognizes a hyphen. */ - SO_O_NOSLASH = 0x0002, - - /*! Permit arguments on single letter options with no equals sign. - e.g. -oARG or -o[ARG] */ - SO_O_SHORTARG = 0x0004, - - /*! Permit single character options to be clumped into a single - option string. e.g. "-a -b -c" <==> "-abc" */ - SO_O_CLUMP = 0x0008, - - /*! Process the entire argv array for options, including the - argv[0] entry. */ - SO_O_USEALL = 0x0010, - - /*! Do not generate an error for invalid options. errors for missing - arguments will still be generated. invalid options will be - treated as files. invalid options in clumps will be silently - ignored. */ - SO_O_NOERR = 0x0020, - - /*! Validate argument type pedantically. Return an error when a - separated argument "-opt arg" is supplied by the user as a - combined argument "-opt=arg". By default this is not considered - an error. */ - SO_O_PEDANTIC = 0x0040, - - /*! Case-insensitive comparisons for short arguments */ - SO_O_ICASE_SHORT = 0x0100, - - /*! Case-insensitive comparisons for long arguments */ - SO_O_ICASE_LONG = 0x0200, - - /*! Case-insensitive comparisons for word arguments - i.e. arguments without any hyphens at the start. */ - SO_O_ICASE_WORD = 0x0400, - - /*! Case-insensitive comparisons for all arg types */ - SO_O_ICASE = 0x0700 -}; - -/*! Types of arguments that options may have. Note that some of the _ESOFlags - are not compatible with all argument types. SO_O_SHORTARG requires that - relevant options use either SO_REQ_CMB or SO_OPT. SO_O_CLUMP requires - that relevant options use only SO_NONE. - */ -typedef enum _ESOArgType { - /*! No argument. Just the option flags. - e.g. -o --opt */ - SO_NONE, - - /*! Required separate argument. - e.g. -o ARG --opt ARG */ - SO_REQ_SEP, - - /*! Required combined argument. - e.g. -oARG -o=ARG --opt=ARG */ - SO_REQ_CMB, - - /*! Optional combined argument. - e.g. -o[ARG] -o[=ARG] --opt[=ARG] */ - SO_OPT, - - /*! Multiple separate arguments. The actual number of arguments is - determined programatically at the time the argument is processed. - e.g. -o N ARG1 ARG2 ... ARGN --opt N ARG1 ARG2 ... ARGN */ - SO_MULTI -} ESOArgType; - -//! this option definition must be the last entry in the table -#define SO_END_OF_OPTIONS { -1, NULL, SO_NONE } - -#ifdef _DEBUG -# ifdef _MSC_VER -# include -# define SO_ASSERT(b) _ASSERTE(b) -# else -# include -# define SO_ASSERT(b) assert(b) -# endif -#else -# define SO_ASSERT(b) //!< assertion used to test input data -#endif - -// --------------------------------------------------------------------------- -// MAIN TEMPLATE CLASS -// --------------------------------------------------------------------------- - -/*! @brief Implementation of the SimpleOpt class */ -template -class CSimpleOptTempl -{ -public: - /*! @brief Structure used to define all known options. */ - struct SOption { - /*! ID to return for this flag. Optional but must be >= 0 */ - int nId; - - /*! arg string to search for, e.g. "open", "-", "-f", "--file" - Note that on Windows the slash option marker will be converted - to a hyphen so that "-f" will also match "/f". */ - const SOCHAR * pszArg; - - /*! type of argument accepted by this option */ - ESOArgType nArgType; - }; - - /*! @brief Initialize the class. Init() must be called later. */ - CSimpleOptTempl() - : m_rgShuffleBuf(NULL) - { - Init(0, NULL, NULL, 0); - } - - /*! @brief Initialize the class in preparation for use. */ - CSimpleOptTempl( - int argc, - SOCHAR * argv[], - const SOption * a_rgOptions, - int a_nFlags = 0 - ) - : m_rgShuffleBuf(NULL) - { - Init(argc, argv, a_rgOptions, a_nFlags); - } - -#ifndef SO_MAX_ARGS - /*! @brief Deallocate any allocated memory. */ - ~CSimpleOptTempl() { if (m_rgShuffleBuf) free(m_rgShuffleBuf); } -#endif - - /*! @brief Initialize the class in preparation for calling Next. - - The table of options pointed to by a_rgOptions does not need to be - valid at the time that Init() is called. However on every call to - Next() the table pointed to must be a valid options table with the - last valid entry set to SO_END_OF_OPTIONS. - - NOTE: the array pointed to by a_argv will be modified by this - class and must not be used or modified outside of member calls to - this class. - - @param a_argc Argument array size - @param a_argv Argument array - @param a_rgOptions Valid option array - @param a_nFlags Optional flags to modify the processing of - the arguments - - @return true Successful - @return false if SO_MAX_ARGC > 0: Too many arguments - if SO_MAX_ARGC == 0: Memory allocation failure - */ - bool Init( - int a_argc, - SOCHAR * a_argv[], - const SOption * a_rgOptions, - int a_nFlags = 0 - ); - - /*! @brief Change the current options table during option parsing. - - @param a_rgOptions Valid option array - */ - inline void SetOptions(const SOption * a_rgOptions) { - m_rgOptions = a_rgOptions; - } - - /*! @brief Change the current flags during option parsing. - - Note that changing the SO_O_USEALL flag here will have no affect. - It must be set using Init() or the constructor. - - @param a_nFlags Flags to modify the processing of the arguments - */ - inline void SetFlags(int a_nFlags) { m_nFlags = a_nFlags; } - - /*! @brief Query if a particular flag is set */ - inline bool HasFlag(int a_nFlag) const { - return (m_nFlags & a_nFlag) == a_nFlag; - } - - /*! @brief Advance to the next option if available. - - When all options have been processed it will return false. When true - has been returned, you must check for an invalid or unrecognized - option using the LastError() method. This will be return an error - value other than SO_SUCCESS on an error. All standard data - (e.g. OptionText(), OptionArg(), OptionId(), etc) will be available - depending on the error. - - After all options have been processed, the remaining files from the - command line can be processed in same order as they were passed to - the program. - - @return true option or error available for processing - @return false all options have been processed - */ - bool Next(); - - /*! Stops processing of the command line and returns all remaining - arguments as files. The next call to Next() will return false. - */ - void Stop(); - - /*! @brief Return the last error that occurred. - - This function must always be called before processing the current - option. This function is available only when Next() has returned true. - */ - inline ESOError LastError() const { return m_nLastError; } - - /*! @brief Return the nId value from the options array for the current - option. - - This function is available only when Next() has returned true. - */ - inline int OptionId() const { return m_nOptionId; } - - /*! @brief Return the pszArg from the options array for the current - option. - - This function is available only when Next() has returned true. - */ - inline const SOCHAR * OptionText() const { return m_pszOptionText; } - - /*! @brief Return the argument for the current option where one exists. - - If there is no argument for the option, this will return NULL. - This function is available only when Next() has returned true. - */ - inline SOCHAR * OptionArg() const { return m_pszOptionArg; } - - /*! @brief Validate and return the desired number of arguments. - - This is only valid when OptionId() has return the ID of an option - that is registered as SO_MULTI. It may be called multiple times - each time returning the desired number of arguments. Previously - returned argument pointers are remain valid. - - If an error occurs during processing, NULL will be returned and - the error will be available via LastError(). - - @param n Number of arguments to return. - */ - SOCHAR ** MultiArg(int n); - - /*! @brief Returned the number of entries in the Files() array. - - After Next() has returned false, this will be the list of files (or - otherwise unprocessed arguments). - */ - inline int FileCount() const { return m_argc - m_nLastArg; } - - /*! @brief Return the specified file argument. - - @param n Index of the file to return. This must be between 0 - and FileCount() - 1; - */ - inline SOCHAR * File(int n) const { - SO_ASSERT(n >= 0 && n < FileCount()); - return m_argv[m_nLastArg + n]; - } - - /*! @brief Return the array of files. */ - inline SOCHAR ** Files() const { return &m_argv[m_nLastArg]; } - -private: - CSimpleOptTempl(const CSimpleOptTempl &); // disabled - CSimpleOptTempl & operator=(const CSimpleOptTempl &); // disabled - - SOCHAR PrepareArg(SOCHAR * a_pszString) const; - bool NextClumped(); - void ShuffleArg(int a_nStartIdx, int a_nCount); - int LookupOption(const SOCHAR * a_pszOption) const; - int CalcMatch(const SOCHAR *a_pszSource, const SOCHAR *a_pszTest) const; - - // Find the '=' character within a string. - inline SOCHAR * FindEquals(SOCHAR *s) const { - while (*s && *s != (SOCHAR)'=') ++s; - return *s ? s : NULL; - } - bool IsEqual(SOCHAR a_cLeft, SOCHAR a_cRight, int a_nArgType) const; - - inline void Copy(SOCHAR ** ppDst, SOCHAR ** ppSrc, int nCount) const { -#ifdef SO_MAX_ARGS - // keep our promise of no CLIB usage - while (nCount-- > 0) *ppDst++ = *ppSrc++; -#else - memcpy(ppDst, ppSrc, nCount * sizeof(SOCHAR*)); -#endif - } - -private: - const SOption * m_rgOptions; //!< pointer to options table - int m_nFlags; //!< flags - int m_nOptionIdx; //!< current argv option index - int m_nOptionId; //!< id of current option (-1 = invalid) - int m_nNextOption; //!< index of next option - int m_nLastArg; //!< last argument, after this are files - int m_argc; //!< argc to process - SOCHAR ** m_argv; //!< argv - const SOCHAR * m_pszOptionText; //!< curr option text, e.g. "-f" - SOCHAR * m_pszOptionArg; //!< curr option arg, e.g. "c:\file.txt" - SOCHAR * m_pszClump; //!< clumped single character options - SOCHAR m_szShort[3]; //!< temp for clump and combined args - ESOError m_nLastError; //!< error status from the last call - SOCHAR ** m_rgShuffleBuf; //!< shuffle buffer for large argc -}; - -// --------------------------------------------------------------------------- -// IMPLEMENTATION -// --------------------------------------------------------------------------- - -template -bool -CSimpleOptTempl::Init( - int a_argc, - SOCHAR * a_argv[], - const SOption * a_rgOptions, - int a_nFlags - ) -{ - m_argc = a_argc; - m_nLastArg = a_argc; - m_argv = a_argv; - m_rgOptions = a_rgOptions; - m_nLastError = SO_SUCCESS; - m_nOptionIdx = 0; - m_nOptionId = -1; - m_pszOptionText = NULL; - m_pszOptionArg = NULL; - m_nNextOption = (a_nFlags & SO_O_USEALL) ? 0 : 1; - m_szShort[0] = (SOCHAR)'-'; - m_szShort[2] = (SOCHAR)'\0'; - m_nFlags = a_nFlags; - m_pszClump = NULL; - -#ifdef SO_MAX_ARGS - if (m_argc > SO_MAX_ARGS) { - m_nLastError = SO_ARG_INVALID_DATA; - m_nLastArg = 0; - return false; - } -#else - if (m_rgShuffleBuf) { - free(m_rgShuffleBuf); - } - if (m_argc > SO_STATICBUF) { - m_rgShuffleBuf = (SOCHAR**) malloc(sizeof(SOCHAR*) * m_argc); - if (!m_rgShuffleBuf) { - return false; - } - } -#endif - - return true; -} - -template -bool -CSimpleOptTempl::Next() -{ -#ifdef SO_MAX_ARGS - if (m_argc > SO_MAX_ARGS) { - SO_ASSERT(!"Too many args! Check the return value of Init()!"); - return false; - } -#endif - - // process a clumped option string if appropriate - if (m_pszClump && *m_pszClump) { - // silently discard invalid clumped option - bool bIsValid = NextClumped(); - while (*m_pszClump && !bIsValid && HasFlag(SO_O_NOERR)) { - bIsValid = NextClumped(); - } - - // return this option if valid or we are returning errors - if (bIsValid || !HasFlag(SO_O_NOERR)) { - return true; - } - } - SO_ASSERT(!m_pszClump || !*m_pszClump); - m_pszClump = NULL; - - // init for the next option - m_nOptionIdx = m_nNextOption; - m_nOptionId = -1; - m_pszOptionText = NULL; - m_pszOptionArg = NULL; - m_nLastError = SO_SUCCESS; - - // find the next option - SOCHAR cFirst; - int nTableIdx = -1; - int nOptIdx = m_nOptionIdx; - while (nTableIdx < 0 && nOptIdx < m_nLastArg) { - SOCHAR * pszArg = m_argv[nOptIdx]; - m_pszOptionArg = NULL; - - // find this option in the options table - cFirst = PrepareArg(pszArg); - if (pszArg[0] == (SOCHAR)'-') { - // find any combined argument string and remove equals sign - m_pszOptionArg = FindEquals(pszArg); - if (m_pszOptionArg) { - *m_pszOptionArg++ = (SOCHAR)'\0'; - } - } - nTableIdx = LookupOption(pszArg); - - // if we didn't find this option but if it is a short form - // option then we try the alternative forms - if (nTableIdx < 0 - && !m_pszOptionArg - && pszArg[0] == (SOCHAR)'-' - && pszArg[1] - && pszArg[1] != (SOCHAR)'-' - && pszArg[2]) - { - // test for a short-form with argument if appropriate - if (HasFlag(SO_O_SHORTARG)) { - m_szShort[1] = pszArg[1]; - int nIdx = LookupOption(m_szShort); - if (nIdx >= 0 - && (m_rgOptions[nIdx].nArgType == SO_REQ_CMB - || m_rgOptions[nIdx].nArgType == SO_OPT)) - { - m_pszOptionArg = &pszArg[2]; - pszArg = m_szShort; - nTableIdx = nIdx; - } - } - - // test for a clumped short-form option string and we didn't - // match on the short-form argument above - if (nTableIdx < 0 && HasFlag(SO_O_CLUMP)) { - m_pszClump = &pszArg[1]; - ++m_nNextOption; - if (nOptIdx > m_nOptionIdx) { - ShuffleArg(m_nOptionIdx, nOptIdx - m_nOptionIdx); - } - return Next(); - } - } - - // The option wasn't found. If it starts with a switch character - // and we are not suppressing errors for invalid options then it - // is reported as an error, otherwise it is data. - if (nTableIdx < 0) { - if (!HasFlag(SO_O_NOERR) && pszArg[0] == (SOCHAR)'-') { - m_pszOptionText = pszArg; - break; - } - - pszArg[0] = cFirst; - ++nOptIdx; - if (m_pszOptionArg) { - *(--m_pszOptionArg) = (SOCHAR)'='; - } - } - } - - // end of options - if (nOptIdx >= m_nLastArg) { - if (nOptIdx > m_nOptionIdx) { - ShuffleArg(m_nOptionIdx, nOptIdx - m_nOptionIdx); - } - return false; - } - ++m_nNextOption; - - // get the option id - ESOArgType nArgType = SO_NONE; - if (nTableIdx < 0) { - m_nLastError = (ESOError) nTableIdx; // error code - } - else { - m_nOptionId = m_rgOptions[nTableIdx].nId; - m_pszOptionText = m_rgOptions[nTableIdx].pszArg; - - // ensure that the arg type is valid - nArgType = m_rgOptions[nTableIdx].nArgType; - switch (nArgType) { - case SO_NONE: - if (m_pszOptionArg) { - m_nLastError = SO_ARG_INVALID; - } - break; - - case SO_REQ_SEP: - if (m_pszOptionArg) { - // they wanted separate args, but we got a combined one, - // unless we are pedantic, just accept it. - if (HasFlag(SO_O_PEDANTIC)) { - m_nLastError = SO_ARG_INVALID_TYPE; - } - } - // more processing after we shuffle - break; - - case SO_REQ_CMB: - if (!m_pszOptionArg) { - m_nLastError = SO_ARG_MISSING; - } - break; - - case SO_OPT: - // nothing to do - break; - - case SO_MULTI: - // nothing to do. Caller must now check for valid arguments - // using GetMultiArg() - break; - } - } - - // shuffle the files out of the way - if (nOptIdx > m_nOptionIdx) { - ShuffleArg(m_nOptionIdx, nOptIdx - m_nOptionIdx); - } - - // we need to return the separate arg if required, just re-use the - // multi-arg code because it all does the same thing - if ( nArgType == SO_REQ_SEP - && !m_pszOptionArg - && m_nLastError == SO_SUCCESS) - { - SOCHAR ** ppArgs = MultiArg(1); - if (ppArgs) { - m_pszOptionArg = *ppArgs; - } - } - - return true; -} - -template -void -CSimpleOptTempl::Stop() -{ - if (m_nNextOption < m_nLastArg) { - ShuffleArg(m_nNextOption, m_nLastArg - m_nNextOption); - } -} - -template -SOCHAR -CSimpleOptTempl::PrepareArg( - SOCHAR * a_pszString - ) const -{ -#ifdef _WIN32 - // On Windows we can accept the forward slash as a single character - // option delimiter, but it cannot replace the '-' option used to - // denote stdin. On Un*x paths may start with slash so it may not - // be used to start an option. - if (!HasFlag(SO_O_NOSLASH) - && a_pszString[0] == (SOCHAR)'/' - && a_pszString[1] - && a_pszString[1] != (SOCHAR)'-') - { - a_pszString[0] = (SOCHAR)'-'; - return (SOCHAR)'/'; - } -#endif - return a_pszString[0]; -} - -template -bool -CSimpleOptTempl::NextClumped() -{ - // prepare for the next clumped option - m_szShort[1] = *m_pszClump++; - m_nOptionId = -1; - m_pszOptionText = NULL; - m_pszOptionArg = NULL; - m_nLastError = SO_SUCCESS; - - // lookup this option, ensure that we are using exact matching - int nSavedFlags = m_nFlags; - m_nFlags = SO_O_EXACT; - int nTableIdx = LookupOption(m_szShort); - m_nFlags = nSavedFlags; - - // unknown option - if (nTableIdx < 0) { - m_nLastError = (ESOError) nTableIdx; // error code - return false; - } - - // valid option - m_pszOptionText = m_rgOptions[nTableIdx].pszArg; - ESOArgType nArgType = m_rgOptions[nTableIdx].nArgType; - if (nArgType == SO_NONE) { - m_nOptionId = m_rgOptions[nTableIdx].nId; - return true; - } - - if (nArgType == SO_REQ_CMB && *m_pszClump) { - m_nOptionId = m_rgOptions[nTableIdx].nId; - m_pszOptionArg = m_pszClump; - while (*m_pszClump) ++m_pszClump; // must point to an empty string - return true; - } - - // invalid option as it requires an argument - m_nLastError = SO_ARG_MISSING; - return true; -} - -// Shuffle arguments to the end of the argv array. -// -// For example: -// argv[] = { "0", "1", "2", "3", "4", "5", "6", "7", "8" }; -// -// ShuffleArg(1, 1) = { "0", "2", "3", "4", "5", "6", "7", "8", "1" }; -// ShuffleArg(5, 2) = { "0", "1", "2", "3", "4", "7", "8", "5", "6" }; -// ShuffleArg(2, 4) = { "0", "1", "6", "7", "8", "2", "3", "4", "5" }; -template -void -CSimpleOptTempl::ShuffleArg( - int a_nStartIdx, - int a_nCount - ) -{ - SOCHAR * staticBuf[SO_STATICBUF]; - SOCHAR ** buf = m_rgShuffleBuf ? m_rgShuffleBuf : staticBuf; - int nTail = m_argc - a_nStartIdx - a_nCount; - - // make a copy of the elements to be moved - Copy(buf, m_argv + a_nStartIdx, a_nCount); - - // move the tail down - Copy(m_argv + a_nStartIdx, m_argv + a_nStartIdx + a_nCount, nTail); - - // append the moved elements to the tail - Copy(m_argv + a_nStartIdx + nTail, buf, a_nCount); - - // update the index of the last unshuffled arg - m_nLastArg -= a_nCount; -} - -// match on the long format strings. partial matches will be -// accepted only if that feature is enabled. -template -int -CSimpleOptTempl::LookupOption( - const SOCHAR * a_pszOption - ) const -{ - int nBestMatch = -1; // index of best match so far - int nBestMatchLen = 0; // matching characters of best match - int nLastMatchLen = 0; // matching characters of last best match - - for (int n = 0; m_rgOptions[n].nId >= 0; ++n) { - // the option table must use hyphens as the option character, - // the slash character is converted to a hyphen for testing. - SO_ASSERT(m_rgOptions[n].pszArg[0] != (SOCHAR)'/'); - - int nMatchLen = CalcMatch(m_rgOptions[n].pszArg, a_pszOption); - if (nMatchLen == -1) { - return n; - } - if (nMatchLen > 0 && nMatchLen >= nBestMatchLen) { - nLastMatchLen = nBestMatchLen; - nBestMatchLen = nMatchLen; - nBestMatch = n; - } - } - - // only partial matches or no match gets to here, ensure that we - // don't return a partial match unless it is a clear winner - if (HasFlag(SO_O_EXACT) || nBestMatch == -1) { - return SO_OPT_INVALID; - } - return (nBestMatchLen > nLastMatchLen) ? nBestMatch : SO_OPT_MULTIPLE; -} - -// calculate the number of characters that match (case-sensitive) -// 0 = no match, > 0 == number of characters, -1 == perfect match -template -int -CSimpleOptTempl::CalcMatch( - const SOCHAR * a_pszSource, - const SOCHAR * a_pszTest - ) const -{ - if (!a_pszSource || !a_pszTest) { - return 0; - } - - // determine the argument type - int nArgType = SO_O_ICASE_LONG; - if (a_pszSource[0] != '-') { - nArgType = SO_O_ICASE_WORD; - } - else if (a_pszSource[1] != '-' && !a_pszSource[2]) { - nArgType = SO_O_ICASE_SHORT; - } - - // match and skip leading hyphens - while (*a_pszSource == (SOCHAR)'-' && *a_pszSource == *a_pszTest) { - ++a_pszSource; - ++a_pszTest; - } - if (*a_pszSource == (SOCHAR)'-' || *a_pszTest == (SOCHAR)'-') { - return 0; - } - - // find matching number of characters in the strings - int nLen = 0; - while (*a_pszSource && IsEqual(*a_pszSource, *a_pszTest, nArgType)) { - ++a_pszSource; - ++a_pszTest; - ++nLen; - } - - // if we have exhausted the source... - if (!*a_pszSource) { - // and the test strings, then it's a perfect match - if (!*a_pszTest) { - return -1; - } - - // otherwise the match failed as the test is longer than - // the source. i.e. "--mant" will not match the option "--man". - return 0; - } - - // if we haven't exhausted the test string then it is not a match - // i.e. "--mantle" will not best-fit match to "--mandate" at all. - if (*a_pszTest) { - return 0; - } - - // partial match to the current length of the test string - return nLen; -} - -template -bool -CSimpleOptTempl::IsEqual( - SOCHAR a_cLeft, - SOCHAR a_cRight, - int a_nArgType - ) const -{ - // if this matches then we are doing case-insensitive matching - if (m_nFlags & a_nArgType) { - if (a_cLeft >= 'A' && a_cLeft <= 'Z') a_cLeft += 'a' - 'A'; - if (a_cRight >= 'A' && a_cRight <= 'Z') a_cRight += 'a' - 'A'; - } - return a_cLeft == a_cRight; -} - -// calculate the number of characters that match (case-sensitive) -// 0 = no match, > 0 == number of characters, -1 == perfect match -template -SOCHAR ** -CSimpleOptTempl::MultiArg( - int a_nCount - ) -{ - // ensure we have enough arguments - if (m_nNextOption + a_nCount > m_nLastArg) { - m_nLastError = SO_ARG_MISSING; - return NULL; - } - - // our argument array - SOCHAR ** rgpszArg = &m_argv[m_nNextOption]; - - // Ensure that each of the following don't start with an switch character. - // Only make this check if we are returning errors for unknown arguments. - if (!HasFlag(SO_O_NOERR)) { - for (int n = 0; n < a_nCount; ++n) { - SOCHAR ch = PrepareArg(rgpszArg[n]); - if (rgpszArg[n][0] == (SOCHAR)'-') { - rgpszArg[n][0] = ch; - m_nLastError = SO_ARG_INVALID_DATA; - return NULL; - } - rgpszArg[n][0] = ch; - } - } - - // all good - m_nNextOption += a_nCount; - return rgpszArg; -} - - -// --------------------------------------------------------------------------- -// TYPE DEFINITIONS -// --------------------------------------------------------------------------- - -/*! @brief ASCII/MBCS version of CSimpleOpt */ -typedef CSimpleOptTempl CSimpleOptA; - -/*! @brief wchar_t version of CSimpleOpt */ -typedef CSimpleOptTempl CSimpleOptW; - -#if defined(_UNICODE) -/*! @brief TCHAR version dependent on if _UNICODE is defined */ -# define CSimpleOpt CSimpleOptW -#else -/*! @brief TCHAR version dependent on if _UNICODE is defined */ -# define CSimpleOpt CSimpleOptA -#endif - -#endif // INCLUDED_SimpleOpt diff --git a/include/common.hpp b/include/common.hpp new file mode 100644 index 0000000..4607c03 --- /dev/null +++ b/include/common.hpp @@ -0,0 +1,14 @@ +#ifndef __COMMON_HPP__ +#define __COMMON_HPP__ + +#include +#include "plog/Log.h" +#include "mplog/ColorConsoleAppenderStderr.h" +// #include "plog/Formatters/FuncMessageFormatter.h" + +static const std::string stormexVersion = "2.0.0"; + +static plog::ColorConsoleAppenderStdErr consoleAppender; +// static plog::ColorConsoleAppender consoleAppender; + +#endif // __COMMON_HPP__ diff --git a/include/mplog/ColorConsoleAppenderStderr.h b/include/mplog/ColorConsoleAppenderStderr.h new file mode 100644 index 0000000..6e10823 --- /dev/null +++ b/include/mplog/ColorConsoleAppenderStderr.h @@ -0,0 +1,103 @@ +// https://github.com/SergiusTheBest/plog/issues/117 + +#pragma once +#include "mplog/ConsoleAppenderStderr.h" +#include + +namespace plog +{ + template + class ColorConsoleAppenderStdErr : public ConsoleAppenderStderr + { + public: +#ifdef _WIN32 + ColorConsoleAppenderStdErr() : m_originalAttr() + { + if (this->m_isatty) + { + CONSOLE_SCREEN_BUFFER_INFO csbiInfo; + GetConsoleScreenBufferInfo(this->m_stdoutHandle, &csbiInfo); + + m_originalAttr = csbiInfo.wAttributes; + } + } +#else + ColorConsoleAppenderStdErr() {} +#endif + + virtual void write(const Record& record) + { + util::nstring str = Formatter::format(record); + util::MutexLock lock(this->m_mutex); + + setColor(record.getSeverity()); + this->writestr(str); + resetColor(); + } + + private: + void setColor(Severity severity) + { + if (this->m_isatty) + { + switch (severity) + { +#ifdef _WIN32 + case fatal: + SetConsoleTextAttribute(this->m_stdoutHandle, foreground::kRed | foreground::kGreen | foreground::kBlue | foreground::kIntensity | background::kRed); // white on red background + break; + + case error: + SetConsoleTextAttribute(this->m_stdoutHandle, static_cast(foreground::kRed | foreground::kIntensity | (m_originalAttr & 0xf0))); // red + break; + + case warning: + SetConsoleTextAttribute(this->m_stdoutHandle, static_cast(foreground::kRed | foreground::kGreen | foreground::kIntensity | (m_originalAttr & 0xf0))); // yellow + break; + + case debug: + case verbose: + SetConsoleTextAttribute(this->m_stdoutHandle, static_cast(foreground::kGreen | foreground::kBlue | foreground::kIntensity | (m_originalAttr & 0xf0))); // cyan + break; +#else + case fatal: + std::cerr << "\x1B[97m\x1B[41m"; // white on red background + break; + + case error: + std::cerr << "\x1B[91m"; // red + break; + + case warning: + std::cerr << "\x1B[93m"; // yellow + break; + + case debug: + case verbose: + std::cerr << "\x1B[96m"; // cyan + break; +#endif + default: + break; + } + } + } + + void resetColor() + { + if (this->m_isatty) + { +#ifdef _WIN32 + SetConsoleTextAttribute(this->m_stdoutHandle, m_originalAttr); +#else + std::cerr << "\x1B[0m\x1B[0K"; +#endif + } + } + + private: +#ifdef _WIN32 + WORD m_originalAttr; +#endif + }; +} diff --git a/include/mplog/ConsoleAppenderStderr.h b/include/mplog/ConsoleAppenderStderr.h new file mode 100644 index 0000000..b2384ff --- /dev/null +++ b/include/mplog/ConsoleAppenderStderr.h @@ -0,0 +1,64 @@ +// https://github.com/SergiusTheBest/plog/issues/117 + +#pragma once +#include +#include +#include +#include + +namespace plog +{ + template + class ConsoleAppenderStderr : public IAppender + { + public: +#ifdef _WIN32 + ConsoleAppenderStderr() : m_isatty(!!_isatty(_fileno(stderr))), m_stdoutHandle() + { + if (m_isatty) + { + m_stdoutHandle = GetStdHandle(stdHandle::kOutput); + } + } +#else + ConsoleAppenderStderr() : m_isatty(!!isatty(fileno(stderr))) {} +#endif + + virtual void write(const Record& record) + { + util::nstring str = Formatter::format(record); + util::MutexLock lock(m_mutex); + + writestr(str); + } + + protected: + void writestr(const util::nstring& str) + { +#ifdef _WIN32 + if (m_isatty) + { + WriteConsoleW(m_stdoutHandle, str.c_str(), static_cast(str.size()), NULL, NULL); + } + else + { + std::cerr << util::toNarrow(str, codePage::kActive) << std::flush; + } +#else + std::cerr << str << std::flush; +#endif + } + + private: +#ifdef __BORLANDC__ + static int _isatty(int fd) { return ::isatty(fd); } +#endif + + protected: + util::Mutex m_mutex; + const bool m_isatty; +#ifdef _WIN32 + HANDLE m_stdoutHandle; +#endif + }; +} diff --git a/include/storage.hpp b/include/storage.hpp new file mode 100644 index 0000000..738e707 --- /dev/null +++ b/include/storage.hpp @@ -0,0 +1,68 @@ +#ifndef __STORAGE_HPP__ +#define __STORAGE_HPP__ + +#include +#include +#include +#include + +#define __CASCLIB_SELF__ +#include "../CascLib/src/CascLib.h" +#include "common.hpp" +#include "util.hpp" + +struct StorageSearchResult +{ + size_t fileSize; +}; + +/** + * @brief CASC Storage Explorer + */ +class StorageExplorer { +protected: + HANDLE m_hStorage = nullptr; + +public: + const HANDLE& getHandle() { return m_hStorage; } + + /** + * @brief Open CASC + * + * @param src + * @return non zero in case of failure + */ + int openStorage(std::string src); + + /** + * @brief Close CASC + * + * @return true + * @return false + */ + bool closeStorage(); + + // TODO: CascGetStorageInfo + + std::vector enumerateFiles(); + + /** + * @brief extract data of given file to location specified under filesystem + * + * @param storedFilename + * @param targetFilename + * @return size_t + */ + size_t extractFileToPath(const std::string& storedFilename, const std::string& targetFilename); + + /** + * @brief extract data of given file and write it to a FILE stream (not limited to files) + * + * @param storedFilename + * @param outStream + * @return size_t + */ + size_t extractFileData(const std::string& storedFilename, FILE* outStream); +}; + +#endif // __STORAGE_HPP__ diff --git a/include/util.hpp b/include/util.hpp new file mode 100644 index 0000000..380e849 --- /dev/null +++ b/include/util.hpp @@ -0,0 +1,26 @@ +#ifndef __UTIL_HPP__ +#define __UTIL_HPP__ + +#include + +#if (defined(_WIN32) || defined(_WIN64)) + #include + #define mkdir(name, chmod) _mkdir(name) +#endif +#if defined(WIN32) || defined(_WIN32) + #define PATH_SEP_STR "\\" + #define PATH_SEP_CHAR '\\' +#else + #define PATH_SEP_STR "/" + #define PATH_SEP_CHAR '/' +#endif + +bool pathExists(const std::string& target); +int ensureDirExists(std::string strDestName); + +std::string formatFileSize(size_t size); + +/// Try to find in the Haystack the Needle - ignore case +bool findStringIC(const std::string& strHaystack, const std::string& strNeedle); + +#endif // __UTIL_HPP__ diff --git a/plog b/plog new file mode 160000 index 0000000..fcf13b4 --- /dev/null +++ b/plog @@ -0,0 +1 @@ +Subproject commit fcf13b4d736d121c9c6bf0ea01fd160660f14fcf diff --git a/src/storage.cc b/src/storage.cc new file mode 100644 index 0000000..abe1c15 --- /dev/null +++ b/src/storage.cc @@ -0,0 +1,86 @@ +#include +#include "storage.hpp" + +int StorageExplorer::openStorage(std::string src) +{ + // Remove trailing slash at the end of path (CascLib doesn't like that, supposedly) + if (src[src.size() - 1] == '/' || src[src.size() - 1] == '\\') { + src = src.substr(0, src.size() - 1); + } + + if (!CascOpenStorage(src.c_str(), 0, &m_hStorage)) { + return GetLastError(); + } + + return 0; +} + +bool StorageExplorer::closeStorage() +{ + return CascCloseStorage(m_hStorage); +} + +std::vector StorageExplorer::enumerateFiles() +{ + std::vector ret; + + CASC_FIND_DATA findData; + HANDLE handle = CascFindFirstFile(m_hStorage, "*", &findData, NULL); + + do { + findData.dwFileSize; + findData.FileKey; + findData.szFileName; + ret.push_back(findData.szFileName); + } while (CascFindNextFile(handle, &findData)); + + CascFindClose(handle); + + return ret; +} + +size_t StorageExplorer::extractFileToPath(const std::string& storedFilename, const std::string& targetFilename) +{ + int tmp; + + if ((tmp = ensureDirExists(targetFilename)) != 0) { + PLOG_ERROR << "Couldn't create directory path for file: " << targetFilename << " E(" << tmp << ")"; + return 0; + } + + FILE* fileStream = fopen(targetFilename.c_str(), "wb"); + if (fileStream) { + size_t fileSize = extractFileData(storedFilename, fileStream); + fclose(fileStream); + return fileSize; + } + else { + PLOG_ERROR << "Failed to open file for writing: " << targetFilename << " E(" << errno << ")"; + return 0; + } +} + +size_t StorageExplorer::extractFileData(const std::string& storedFilename, FILE* outStream) +{ + char buffer[0x1000]; + + HANDLE hFile; + size_t fileSize = 0; + if (CascOpenFile(m_hStorage, storedFilename.c_str(), CASC_LOCALE_ALL, 0, &hFile)) { + DWORD read; + do { + if (CascReadFile(hFile, &buffer, sizeof(buffer), &read)) { + fwrite(&buffer, read, 1, outStream); + fileSize += read; + } + } while (read > 0); + + CascCloseFile(hFile); + } + else { + PLOG_ERROR << "Failed to extract: " << storedFilename << " to " << static_cast(outStream) << " E(" << errno << ")"; + return 0; + } + + return fileSize; +} diff --git a/src/storm-extract.cpp b/src/storm-extract.cpp deleted file mode 100644 index ac4f5ee..0000000 --- a/src/storm-extract.cpp +++ /dev/null @@ -1,407 +0,0 @@ -#define __CASCLIB_SELF__ -#include "../CascLib/src/CascLib.h" -#include "../include/SimpleOpt.h" - -#include -#include -#include -#include -#include - -#if (defined(_WIN32) || defined(_WIN64)) - #include - #define mkdir(name, chmod) _mkdir(name) -#endif - -#if defined(WIN32) || defined(_WIN32) - #define PATH_SEP_STR "\\" - #define PATH_SEP_CHAR '\\' -#else - #define PATH_SEP_STR "/" - #define PATH_SEP_CHAR '/' -#endif - -#include -#include -#include -#include -#include - -using namespace std; - - -// All the global variables -string version = "1.4.0"; - -struct tSearchResult { - string strFileName; - string strFullPath; - DWORD lFileSize; -}; - -// Valid options -enum { - OPT_HELP, - OPT_VERBOSE, - OPT_QUIET, - OPT_EXTRACT, - OPT_DEST, - OPT_SEARCH, - OPT_LOWERCASE, - OPT_REGEX_INCLUDE, - OPT_REGEX_EXCLUDE, - OPT_REGEX_ICASE -}; - -HANDLE hStorage; -string strSearchPattern = ""; -std::vector includePatterns; -std::vector excludePatterns; -bool icasePatterns = false; -string strSource; -string strDestination = "."; -bool bExtract = false; -bool bVerbose = false; // Print extra information for logging -bool bQuiet = false; // Do not print anything. - -const CSimpleOpt::SOption COMMAND_LINE_OPTIONS[] = { - { OPT_HELP, "-h", SO_NONE }, - { OPT_HELP, "--help", SO_NONE }, - { OPT_VERBOSE, "-v", SO_NONE }, - { OPT_VERBOSE, "--verbose", SO_NONE }, - { OPT_QUIET, "-q", SO_NONE }, - { OPT_QUIET, "--quiet", SO_NONE }, - { OPT_EXTRACT, "-x", SO_NONE }, - { OPT_EXTRACT, "--extract", SO_NONE }, - { OPT_DEST, "-o", SO_REQ_SEP }, - { OPT_DEST, "--out", SO_REQ_SEP }, - //{ OPT_LOWERCASE, "-c", SO_NONE }, - //{ OPT_LOWERCASE, "--lowercase", SO_NONE }, - { OPT_SEARCH, "-s", SO_REQ_SEP }, - { OPT_SEARCH, "--search", SO_REQ_SEP }, - { OPT_REGEX_ICASE, "--ignore-case", SO_NONE }, - { OPT_REGEX_INCLUDE, "--include", SO_REQ_SEP }, - { OPT_REGEX_EXCLUDE, "--exclude", SO_REQ_SEP }, - - SO_END_OF_OPTIONS -}; - -/* FUNCTIONS */ -void showUsage(const std::string &pathToExecutable) { - cout << "stormex v" << version << endl - << " Usage: " << pathToExecutable << " [options]" << endl - << endl - << "This program can list and optionally extract files from a CASC storage container." << endl - << endl - << " -h, --help Display this help" << endl - << endl - << "Arguments:" << endl - << " Path to game installation folder" << endl - << endl - << "Options:" << endl - << " General:" << endl - << " -v, --verbose Prints more information" << endl - << " -q, --quiet Prints nothing, nada, zip" << endl - << endl - << " Common:" << endl - << " -s, --search Restrict results to full paths matching STRING" << endl - << " --ignore-case Case-insensitive pattern" << endl - << " --include Include files matching ECMAScript regex PATTERN" << endl - << " --exclude Exclude files matching ECMAScript regex PATTERN" << endl - << endl - << " Extract:" << endl - << " -x, --extract Extract the files found" << endl - << " -o, --out The folder where the files are extracted (extract only)" << endl - << endl; -} - -// Overloaded echo command. -void echo() { - if (!bQuiet) { - cout << endl; - } -} - -void echo(const std::string &output) { - if (!bQuiet) { - cout << output; - } -} - -void echo(const int &output) { - if (!bQuiet) { - cout << output; - } -} - -// Overloaded verbose command. -void verbose() { - if (!bQuiet && bVerbose) { - cout << endl; - } -} - -void verbose(const std::string &output) { - if (!bQuiet && bVerbose) { - cout << output; - } -} - -void verbose(const int &output) { - if (!bQuiet && bVerbose) { - cout << output; - } -} - -bool searchRegexMulti(const std::string filename, const std::vector patterns) { - for (size_t i = 0; i < patterns.size(); i++) { - if (regex_search(filename, *patterns[i], regex_constants::match_default)) { - return true; - } - } - - return false; -} - -void printCount( int count, string description ) { - if (!bQuiet) { - std::printf("%c[2K", 27); - std::cout << "\r "; - std::cout.width( 7 ); - std::cout << count << description; - std::cout << std::flush; - } -} - -void printProgress( int percent, string description ) { - if (!bQuiet) { - std::printf("%c[2K", 27); - std::cout << "\r "; - std::cout.width( 6 ); - std::cout << percent << "% " << description; - std::cout << std::flush; - } -} - -vector searchArchive() { - // Instantiate variables - int filesFound = 0; - vector ret; - std::set directoryResults; - std::set::iterator dIter; - - // Let's do dis... - CASC_FIND_DATA findData; - HANDLE handle = CascFindFirstFile(hStorage, "*", &findData, NULL); - - // Looper - if (handle) { - do { - tSearchResult r; - r.strFileName = findData.szPlainName; - r.strFullPath = findData.szFileName; - r.lFileSize = findData.dwFileSize; - - if (r.strFullPath.find(strSearchPattern) != std::string::npos) { - // Apply include/exclude filters - if ( - (includePatterns.size() > 0 && !searchRegexMulti(r.strFullPath, includePatterns)) || - (excludePatterns.size() > 0 && searchRegexMulti(r.strFullPath, excludePatterns)) - ) { - continue; - } - ret.push_back(r.strFullPath); - - if (!bExtract && !bQuiet) { - cout << findData.szFileName << endl; - } - } - } while (CascFindNextFile(handle, &findData) && findData.szPlainName); - - CascFindClose(handle); - } - - return ret; -} - -size_t extractFile(string strFullPath) { - char buffer[0x1000]; - string strDestName = strDestination + strFullPath; - size_t pos; - - // normalize slashes in the path - std::replace(strDestName.begin(), strDestName.end(), '\\', '/'); - - // ensure directory path to the file exists - pos = -1; - while ((pos = strDestName.find('/', pos + 1)) != string::npos) - { - string dirname = strDestName.substr(0, pos); - - DIR* d = opendir(dirname.c_str()); - if (!d) - mkdir(dirname.c_str(), 0755); - else - closedir(d); - } - - // extract data - HANDLE hFile; - size_t fileSize = 0; - if (CascOpenFile(hStorage, strFullPath.c_str(), CASC_LOCALE_ALL, 0, &hFile)) - { - DWORD read; - FILE* dest = fopen(strDestName.c_str(), "wb"); - if (dest) - { - do { - if (CascReadFile(hFile, &buffer, sizeof(buffer), &read)) { - fileSize += fwrite(&buffer, read, 1, dest); - } - } while (read > 0); - - fclose(dest); - } - else - { - cerr << "NOFILE: (" << errno << ") Failed to extract '" << strFullPath << "' to " << strDestName << endl; - return 0; - } - CascCloseFile(hFile); - } - else - { - cerr << "NOARCHIVE: (" << errno << ") Failed to extract '" << strFullPath << "' to " << strDestName << endl; - return 0; - } - return fileSize; -} - -int main(int argc, char** argv) { - - vector searchResults; - std::set directoryResults; - std::set::iterator dIter; - - // Parse the command-line parameters - CSimpleOpt args(argc, argv, COMMAND_LINE_OPTIONS); - while (args.Next()) - { - if (args.LastError() == SO_SUCCESS) - { - switch (args.OptionId()) - { - case OPT_HELP: - showUsage(argv[0]); - return 0; - - case OPT_DEST: - strDestination = args.OptionArg(); - break; - - case OPT_SEARCH: - strSearchPattern = args.OptionArg(); - break; - - // case OPT_LOWERCASE: - // bLowerCase = true; - // break; - - case OPT_QUIET: - bQuiet = true; - break; - - case OPT_VERBOSE: - bVerbose = true; - break; - - case OPT_EXTRACT: - bExtract = true; - break; - - case OPT_REGEX_ICASE: - icasePatterns = true; - break; - - case OPT_REGEX_INCLUDE: - case OPT_REGEX_EXCLUDE: - try { - regex* p = new regex( - args.OptionArg(), - (icasePatterns == true ? regex::ECMAScript | regex::icase : regex::ECMAScript) - ); - if (args.OptionId() == OPT_REGEX_INCLUDE) { - includePatterns.push_back(p); - } - else { - excludePatterns.push_back(p); - } - } catch(const std::regex_error& e) { - std::cerr << e.what() << ": " << args.OptionArg() << endl; - return -1; - } - break; - } - } - else - { - cerr << "Invalid argument: " << args.OptionText() << endl; - return -1; - } - } - - if (!args.FileCount()) { - cerr << "Missing argument " << endl; - return -1; - } - strSource = args.File(0); - - // Remove trailing slashes at the end of the storage path (CascLib doesn't like that) - if ((strSource[strSource.size() - 1] == '/') || (strSource[strSource.size() - 1] == '\\')) - strSource = strSource.substr(0, strSource.size() - 1); - - // Open CASC Files - if (!CascOpenStorage(strSource.c_str(), 0, &hStorage)) { - cerr << "Failed to open the storage '" << strSource << "'" << endl; - return -2; - } - - vector results = searchArchive(); - - if (!bExtract) { - verbose(); - echo(results.size()); - echo(" files found.\n"); - } - - // Extraction - if (bExtract && !results.empty()) - { - int filesDone = 0; - int progress; - echo("Extracting files:\n"); - - if (strDestination.at(strDestination.size() - 1) != PATH_SEP_CHAR) - strDestination += PATH_SEP_STR; - - vector::iterator iter, iterEnd; - for (iter = results.begin(), iterEnd = results.end(); iter != iterEnd; ++iter) - { - filesDone++; - progress = int(filesDone * 100 / results.size()); - printProgress(progress, iter->c_str()); - size_t bytesWritten = extractFile(iter->c_str()); - if (bytesWritten <= 0) { - // TODO: report warning? this might mean file is encrypted - } - verbose(); - } - verbose("\n"); - echo(" "); - echo(filesDone); - echo(" files extracted.\n"); - } - - CascCloseStorage(hStorage); - return 0; -} diff --git a/src/stormex.cc b/src/stormex.cc new file mode 100644 index 0000000..5f42dff --- /dev/null +++ b/src/stormex.cc @@ -0,0 +1,305 @@ +#include +#include +#include +#include +#include +#include +#include + +#include "cxxopts.hpp" +#include "common.hpp" +#include "util.hpp" +#include "storage.hpp" + +class StormexContext { +public: + struct { + std::string storageSrc; + std::string listfileSrc; + } m_base; + + struct { + bool listFiles; + bool showDetails; + } m_list; + + struct { + std::vector searchPhrase; + bool searchSmartCast; + std::vector includePatterns; + std::vector excludePatterns; + } m_filters; + + struct { + std::vector extractFiles; + std::string outDir; + bool stdOut; + bool progress; + bool dryRun; + } m_extract; + + void scanExtraArgs(cxxopts::ParseResult pResult) + { + if (pResult.count("in-regex")) { + parseRegex(m_filters.includePatterns, pResult["in-regex"].as>(), false); + } + if (pResult.count("in-iregex")) { + parseRegex(m_filters.includePatterns, pResult["in-iregex"].as>(), true); + } + if (pResult.count("ex-regex")) { + parseRegex(m_filters.excludePatterns, pResult["ex-regex"].as>(), false); + } + if (pResult.count("ex-iregex")) { + parseRegex(m_filters.excludePatterns, pResult["ex-iregex"].as>(), true); + } + } + +private: + void parseRegex(std::vector& patterns, const std::string input, bool icase) + { + try { + patterns.push_back(std::regex(input, (icase == true ? std::regex::ECMAScript | std::regex::icase : std::regex::ECMAScript))); + } catch (const std::regex_error& e) { + std::cerr << e.what() << ": " << input << std::endl; + exit(-1); + } + } + + void parseRegex(std::vector& patterns, const std::vector input, bool icase) + { + for (const auto& value : input) { + parseRegex(patterns, value, icase); + } + } +}; + +StormexContext appCtx; + +void parseArguments(int argc, char* argv[]) +{ + try { + cxxopts::Options options(argv[0], + "stormex v" + stormexVersion + "\n" + "\n" + "Command-line application to enumerate and extract files from CASC (Content Addressable Storage Container) used in Blizzard games.\n" + "\n" + "Expected regex pattern should follow ECMAScript syntax\n"); + options + .positional_help("[STORAGE]") + .show_positional_help(); + + options.add_options("Common") + ("h,help", "Print help.") + ("v,verbose", "Verbose output.", cxxopts::value()) + ("q,quiet", "Supresses output of messages entirely.", cxxopts::value()) + ("version", "Print version."); + + options.add_options("Base") + ("S,storage", "Path to directory with CASC.", cxxopts::value(appCtx.m_base.storageSrc), "[PATH]") + ("L,listfile", + "Map filenames from provided newline delimeted (LF or CRLF) textfile, instead of enumerating content of the archive, " + "which is an extensive operation. It combines well when extracting single files, or a small group that matches given substring or regex pattern.", cxxopts::value(appCtx.m_base.listfileSrc), "[FILE]"); + + options.add_options("List") + ("l,list", "List files inside CASC.", cxxopts::value(appCtx.m_list.listFiles)) + ("d,details", "Show details about each file - such as its size.", cxxopts::value(appCtx.m_list.showDetails)); + + options.add_options("Filter") + ("s,search", "Search for files using a substring.", cxxopts::value>(appCtx.m_filters.searchPhrase), "[SEARCH...]") + ("smart-case", + "Searches case insensitively if the pattern is all lowercase. Search case sensitively otherwise.", + cxxopts::value(appCtx.m_filters.searchSmartCast)->default_value("true")) + ("i,in-regex", "Include files matching regex.", cxxopts::value>(), "[PATTERN...]") + ("I,in-iregex", "Include files matching regex case insensitively.", cxxopts::value>(), "[PATTERN...]") + ("e,ex-regex", "Exclude files matching regex.", cxxopts::value>(), "[PATTERN...]") + ("E,ex-iregex", "Exclude files matching regex case insensitively.", cxxopts::value>(), "[PATTERN...]"); + + options.add_options("Extract") + ("x,extract", + "Extract file(s) matching exactly. Argument is optional - if ommitted it will extract all files matching search filters.", + cxxopts::value>(appCtx.m_extract.extractFiles), "[FILE...]") + ("O,outdir", "Output directory for extracted files.", cxxopts::value(appCtx.m_extract.outDir)->default_value("./"), "[PATH]") + ("p,stdout", "Pipe content of a file(s) to stdout instead writing it to the filesystem.", cxxopts::value(appCtx.m_extract.stdOut)) + ("P,progress", "Notify about progress during extraction.", cxxopts::value(appCtx.m_extract.progress)) + ("n,dry-run", "Simulate extraction process without writing any data to the filesystem.", cxxopts::value(appCtx.m_extract.dryRun)); + + options.parse_positional({"storage"}); + + auto result = options.parse(argc, argv); + + if (result.count("help")) { + std::cerr << options.help({ "Common", "Base", "List", "Filter", "Extract" }) << std::endl; + exit(0); + } + + if (result.count("version")) { + std::cerr << "stormex v" << stormexVersion << std::endl; + exit(0); + } + + // logging verbosity + plog::Severity level = plog::Severity::warning; + if (result.count("verbose")) { + if (result.count("verbose") >= plog::Severity::debug) { + std::cerr << "Maximum level of logging verbosity has been reached [" << plog::Severity::debug << "]" << std::endl; + exit(1); + } + level = static_cast(plog::Severity::fatal + result.count("verbose")); + } + else if (result.count("quiet")) { + plog::Severity level = plog::Severity::none; + } + plog::init(level, &consoleAppender); + + // validate required arguments + if (!result.count("storage")) { + // TODO: scan positional args + std::cerr << "missing required argument --storage" << std::endl; + exit(1); + } + + appCtx.scanExtraArgs(result); + } catch (const cxxopts::OptionException& e) { + std::cerr << "error parsing options: " << e.what() << std::endl; + exit(1); + } +} + +void extractFiles(StorageExplorer& stExplorer, const std::vector& filesToExtract) +{ + PLOG_DEBUG << "Preparing to extract " << filesToExtract.size() << " files.."; + if (appCtx.m_extract.dryRun) { + PLOG_INFO << "Dry mode is active.."; + } + + if (appCtx.m_extract.stdOut) { + setvbuf(stdout, NULL, _IONBF, 0); + for (const auto& storedFilename : filesToExtract) { + stExplorer.extractFileData(storedFilename, stdout); + } + } + else if (!appCtx.m_extract.outDir.empty()) { + if (!pathExists(appCtx.m_extract.outDir)) { + PLOG_FATAL << "Specified output directory doesn't exist or cannot be opened: " << appCtx.m_extract.outDir; + exit(-3); + } + + PLOG_DEBUG << "Output directory set to: " << appCtx.m_extract.outDir; + + for (const auto& storedFilename : filesToExtract) { + std::string targetFile = appCtx.m_extract.outDir; + if (targetFile.at(targetFile.size() - 1) != PATH_SEP_CHAR) { + targetFile += PATH_SEP_STR; + } + targetFile += storedFilename; + + PLOG_DEBUG << "Extracting file: " << storedFilename; + size_t fileSize = 0; + if (!appCtx.m_extract.dryRun) { + fileSize = stExplorer.extractFileToPath(storedFilename, targetFile); + PLOG_DEBUG << "Saved at: " << targetFile << " [" << formatFileSize(fileSize) << "]"; + } + else { + } + } + } +} + +bool searchRegexMulti(const std::string filename, const std::vector& patterns) +{ + for (const auto& current : patterns) { + if (regex_search(filename, current)) { + return true; + } + } + + return false; +} + +std::vector filterFiles(const std::vector& inputList) +{ + std::vector filteredList; + + for (const auto& entry : inputList) { + if (appCtx.m_filters.searchPhrase.size()) { + bool c = false; + for (const auto& needle : appCtx.m_filters.searchPhrase) { + c = findStringIC(entry, needle); + if (c) break; + } + if (!c) continue; + } + + if (appCtx.m_filters.includePatterns.size() && !searchRegexMulti(entry, appCtx.m_filters.includePatterns)) continue; + if (appCtx.m_filters.excludePatterns.size() && searchRegexMulti(entry, appCtx.m_filters.excludePatterns)) continue; + filteredList.push_back(entry); + } + + return filteredList; +} + +std::vector readListFile(const std::string& filename) +{ + std::vector filelist; + std::ifstream ifs(filename, std::ifstream::in); + + std::string line; + while (std::getline(ifs, line)) { + filelist.push_back(line); + } + + ifs.close(); + + return filelist; +} + +void listFiles(StorageExplorer& stExplorer) +{ + PLOG_INFO << "Enumerating all files in storage.."; + auto inputList = stExplorer.enumerateFiles(); + auto filteredList = inputList; + + if (appCtx.m_filters.searchPhrase.size() || appCtx.m_filters.includePatterns.size() || appCtx.m_filters.excludePatterns.size()) { + PLOG_INFO << "Filtering list.."; + filteredList = filterFiles(inputList); + } + + for (const auto& entry : filteredList) { + std::cout << entry << std::endl; + } + + PLOG_DEBUG << "count " << inputList.size() << " : " << filteredList.size(); +} + +int main(int argc, char* argv[]) +{ + parseArguments(argc, argv); + + StorageExplorer stExplorer; + int tmp; + + PLOG_INFO << "Opening storage.."; + if ((tmp = stExplorer.openStorage(appCtx.m_base.storageSrc)) != 0) { + PLOG_FATAL << "Failed to open the storage: " << appCtx.m_base.storageSrc << " E(" << tmp << ")"; + exit(-1); + } + PLOG_INFO << "Storage opened " << static_cast(stExplorer.getHandle()); + + try { + if (appCtx.m_list.listFiles) { + listFiles(stExplorer); + } + else if (appCtx.m_extract.extractFiles.size()) { + extractFiles(stExplorer, appCtx.m_extract.extractFiles); + } + } catch (const std::exception& e) { + stExplorer.closeStorage(); + PLOG_FATAL << e.what(); + throw; + } + + PLOG_DEBUG << "Closing storage.."; + stExplorer.closeStorage(); + + return 0; +} diff --git a/src/util.cc b/src/util.cc new file mode 100644 index 0000000..173db32 --- /dev/null +++ b/src/util.cc @@ -0,0 +1,93 @@ +#include +#include +#include +#include +#include +#include "util.hpp" + +struct stat info; + +bool pathExists(const std::string& target) +{ + if (stat(target.c_str(), &info) != 0) { + return false; + } + else if (info.st_mode & S_IFDIR) { + return true; + } + else { + return false; + } +} + +int ensureDirExists(std::string strDestName) +{ + // normalize slashes in the paths received from CASC and force '/' + std::replace(strDestName.begin(), strDestName.end(), '\\', '/'); + + // ensure directory path to the file exists + size_t pos = -1; + while ((pos = strDestName.find('/', pos + 1)) != std::string::npos) { + std::string dirname = strDestName.substr(0, pos); + + DIR* d = opendir(dirname.c_str()); + if (!d) { + // we can use '/' liberally here - Windows doesn't seem to care as per + // + // "In Windows NT, both the backslash ( \) and the forward slash (/ ) are valid path delimiters in character strings in run-time routines." + int err = mkdir(dirname.c_str(), 0755); + if (err != 0) { + return err; + } + } + else { + closedir(d); + } + } + + return 0; +} + +template +std::string valueToString(T num) +{ + std::ostringstream convert; + convert << num; + return convert.str(); +} + +static double roundOff(double n) +{ + double d = n * 100.0; + int i = d + 0.5; + d = (float)i / 100.0; + return d; +} + +std::string formatFileSize(size_t size) +{ + static const char *SIZES[] = { "B", "KB", "MB", "GB" }; + int div = 0; + size_t rem = 0; + + while (size >= 1024 && div < (sizeof SIZES / sizeof *SIZES)) { + rem = (size % 1024); + div++; + size /= 1024; + } + + double size_d = (float)size + (float)rem / 1024.0; + std::string result = valueToString(roundOff(size_d)) + " " + SIZES[div]; + return result; +} + +bool findStringIC(const std::string& strHaystack, const std::string& strNeedle) +{ + auto it = std::search( + strHaystack.begin(), strHaystack.end(), + strNeedle.begin(), strNeedle.end(), + [](char ch1, char ch2) { + return std::toupper(ch1) == std::toupper(ch2); + }); + return (it != strHaystack.end()); +} From 2a7ddacb59fb00f42c6cda3263db62e42b8fcde4 Mon Sep 17 00:00:00 2001 From: Talv Date: Fri, 26 Jul 2019 04:59:44 +0200 Subject: [PATCH 2/2] refactored entire codebase.. v2.0.0 --- .editorconfig | 6 +++ .gitmodules | 2 +- CHANGELOG.md | 25 ++++++++++ CascLib | 2 +- README.md | 109 ++++++++++++++++++++++++++++---------------- include/storage.hpp | 38 +++++++++++++-- src/storage.cc | 24 ++++++---- src/stormex.cc | 99 +++++++++++++++++++++++++++------------- src/util.cc | 17 +++---- 9 files changed, 229 insertions(+), 93 deletions(-) create mode 100644 CHANGELOG.md diff --git a/.editorconfig b/.editorconfig index 4d9b62f..797117e 100644 --- a/.editorconfig +++ b/.editorconfig @@ -15,3 +15,9 @@ indent_size = 4 [*.md] trim_trailing_whitespace = false + +[CascLib/**] +trim_trailing_whitespace = false +insert_final_newline = unset +indent_style = unset +end_of_line = unset diff --git a/.gitmodules b/.gitmodules index 578bfec..42b9300 100644 --- a/.gitmodules +++ b/.gitmodules @@ -1,6 +1,6 @@ [submodule "CascLib"] path = CascLib - url = https://github.com/ladislav-zezula/CascLib.git + url = https://github.com/Talv/CascLib.git [submodule "cxxopts"] path = cxxopts url = https://github.com/jarro2783/cxxopts.git diff --git a/CHANGELOG.md b/CHANGELOG.md new file mode 100644 index 0000000..b04449e --- /dev/null +++ b/CHANGELOG.md @@ -0,0 +1,25 @@ +# Change Log + +## [2.0.0] - 2019-07-26 + +* Refactored entire codebase.. +* `CascLib` upgraded to `1.20`: + * Enumerating MNDX storage is now much faster (SC2 & Storm data) + * The library now outputs filepaths with backslashes, instead of forwardslashes as it did previously in case of SC2/Storm. + * Applied [patch](https://github.com/Talv/CascLib/commit/b2646e578b43641a46df5725d951b093a7cefce0) to preserve original case sensitivity of filenames. +* The core functionality remains intact, however some of the existing options/arguments have been renamed and/or reorganized. Hopefully for the better. + +## [1.4.0] - 2019-07-22 + +* Introduced compatibility with Windows + +## [1.3.0] - 2019-05-26 + +* `CascLib` upgraded to [ef66d7bb46f0bb4dd782d3b68eb7dcc358d52a13](https://github.com/ladislav-zezula/CascLib/commit/ef66d7bb46f0bb4dd782d3b68eb7dcc358d52a13). + +## [1.1.0] - 2019-05-26 + +* Forked [storm-extract#216812d7f91ab2ca72b04f2561c587c754273489](https://github.com/nydus/storm-extract/tree/216812d7f91ab2ca72b04f2561c587c754273489). +* Rebranded to `stormex`. +* Removed NodeJS bindings, and everything related. +* Enhanced cli app with regex pattern filters on a filelist. diff --git a/CascLib b/CascLib index ef66d7b..b2646e5 160000 --- a/CascLib +++ b/CascLib @@ -1 +1 @@ -Subproject commit ef66d7bb46f0bb4dd782d3b68eb7dcc358d52a13 +Subproject commit b2646e578b43641a46df5725d951b093a7cefce0 diff --git a/README.md b/README.md index 68e790e..d148173 100644 --- a/README.md +++ b/README.md @@ -1,7 +1,6 @@ # stormex -Command-line application to list and extract files from the [CASC](https://wowdev.wiki/CASC) (Content -Addressable Storage Container) used in Blizzard games. +Command-line application to enumerate and extract files from [CASC](https://wowdev.wiki/CASC) (Content Addressable Storage Container) used in Blizzard games. Tested on: @@ -40,59 +39,89 @@ MSBuild STORMEXTRACT.sln /p:Configuration=Release ## Usage ``` -Usage: stormex [options] - -This program can list and optionally extract files from a CASC storage container. - - -h, --help Display this help +Command-line application to enumerate and extract files from CASC (Content Addressable Storage Container) used in Blizzard games. + +Regex pattern is expected to follow ECMAScript syntax + +Usage: + stormex [OPTION...] [STORAGE] + + Common options: + -h, --help Print help. + -v, --verbose Verbose output. + -q, --quiet Supresses output entirely. + --version Print version. + + Base options: + -S, --storage [PATH] Path to directory with CASC. + + List options: + -l, --list List files inside CASC. + -d, --details Show details about each file - such as its size. + + Filter options: + -s, --search [SEARCH...] Search for files using a substring. + --smart-case Searches case insensitively if the pattern is + all lowercase. Search case sensitively + otherwise. (default: true) + -i, --in-regex [PATTERN...] Include files matching regex. + -I, --in-iregex [PATTERN...] Include files matching regex case + insensitively. + -e, --ex-regex [PATTERN...] Exclude files matching regex. + -E, --ex-iregex [PATTERN...] Exclude files matching regex case + insensitively. + + Extract options: + -x, --extract-all Extract all files matching search filters. + -X, --extract-file [FILE...] Extract file(s) matching exactly. + -o, --outdir [PATH] Output directory for extracted files. + (default: ./) + -p, --stdout Pipe content of a file(s) to stdout instead + writing it to the filesystem. + -P, --progress Notify about progress during extraction. + -n, --dry-run Simulate extraction process without writing + any data to the filesystem. +``` -Arguments: - Path to game installation folder +### Examples -Options: - General: - -v, --verbose Prints more information - -q, --quiet Prints nothing, nada, zip +#### List content - Common: - -s, --search Restrict results to full paths matching STRING - --ignore-case Case-insensitive pattern - --include Include files matching ECMAScript regex PATTERN - --exclude Exclude files matching ECMAScript regex PATTERN +List based on a search phrase - Extract: - -x, --extract Extract the files found - -o, --out The folder where the files are extracted (extract only) +```sh +stormex '/mnt/s1/BnetGameLib/StarCraft II' -s 'buildid' -l ``` -### Examples - -#### List content of CASC based on regex pattern +List all files with details and sort by filesize. ```sh -stormex '/mnt/s1/BnetGameLib/StarCraft II' --include '/BuildId.txt$'` +stormex '/mnt/s1/BnetGameLib/StarCraft II' -ld | sort -h ``` -#### Extract content of CASC based on inclusion and exclusion patterns +#### Extract files based on inclusion and exclusion patterns ```sh -stormex '/mnt/s1/BnetGameLib/StarCraft II' --ignore-case -v \ - --include '\/(DocumentInfo|Objects|Regions|Triggers)$' \ - --include '\.(fx|xml|txt|json|galaxy|SC2Style|SC2Hotkeys|SC2Lib|TriggerLib|SC2Interface|SC2Locale|SC2Components|SC2Layout)$' \ - --exclude '(dede|eses|esmx|frfr|itit|kokr|plpl|ptbr|ruru|zhcn|zhtw)\.sc2data' \ - --exclude '(PreloadAssetDB|TextureReductionValues)\.txt$' \ +stormex '/mnt/s1/BnetGameLib/StarCraft II' \ + -I '\/(DocumentInfo|Objects|Regions|Triggers)$' \ + -I '\.(fx|xml|txt|json|galaxy|SC2Style|SC2Hotkeys|SC2Lib|TriggerLib|SC2Interface|SC2Locale|SC2Components|SC2Layout)$' \ + -E '(dede|eses|esmx|frfr|itit|kokr|plpl|ptbr|ruru|zhcn|zhtw)\.sc2data' \ + -E '(PreloadAssetDB|TextureReductionValues)\.txt$' \ -x -o './out' ``` -## Credits +#### Extract to stdout -The library absolutely, unequivocably, could not be possible without -[ladislav-zezula's CascLib](https://github.com/ladislav-zezula/CascLib) -library. Many thanks to [ladislav-zezula](https://github.com/ladislav-zezula). +Extract specific file to `stdout` and pipe the stream to another program. For example convert dds to png and display it with `imagick`. -~~Most of the program was canibalized from -[Kanma's CASCExtractor](https://github.com/Kanma/CASCExtractor/) with the -purpose of customizing it for Heroes of the Storm and integration into -[NodeJS](https://www.nodejs.org).~~ +```sh +stormex -S '/mnt/s1/BnetGameLib/StarCraft II' -X 'mods/core.sc2mod/base.sc2data/EditorData/Images/HeroesEditor_Logo.tga' -p | display tga: +``` + +```sh +stormex -S '/mnt/s1/BnetGameLib/StarCraft II' -X 'mods/core.sc2mod/base.sc2data/EditorData/Images/EditorLogo.dds' -p | magick dds: png: | display png: +``` + +## Credits -Most of the program was canibalized from [storm-extract](https://github.com/nydus/storm-extract) with the purpose of reducing code complexity by removing NodeJS stuff. And transitioning it into more streamlined cli app. +* Powered by [CascLib](https://github.com/ladislav-zezula/CascLib) diff --git a/include/storage.hpp b/include/storage.hpp index 738e707..4f47e2b 100644 --- a/include/storage.hpp +++ b/include/storage.hpp @@ -11,9 +11,41 @@ #include "common.hpp" #include "util.hpp" -struct StorageSearchResult +// Based on CASC_FIND_DATA +struct STORAGE_SEARCH_RESULT { - size_t fileSize; + // Full name of the found file. In case when this is CKey/EKey, + // this will be just string representation of the key stored in 'FileKey' + std::string filename; + + // Content key. This is present if the CASC_FEATURE_ROOT_CKEY is present + BYTE CKey[MD5_HASH_SIZE]; + + // Encoded key. This is always present. + BYTE EKey[MD5_HASH_SIZE]; + + // Tag mask. Only valid if the storage supports tags, otherwise 0 + ULONGLONG TagBitMask; + + // File data ID. Only valid if the storage supports file data IDs, otherwise CASC_INVALID_ID + DWORD dwFileDataId; + + // Size of the file, as retrieved from CKey entry or EKey entry + DWORD dwFileSize; + + // Locale flags. Only valid if the storage supports locale flags, otherwise CASC_INVALID_ID + DWORD dwLocaleFlags; + + // Content flags. Only valid if the storage supports content flags, otherwise CASC_INVALID_ID + DWORD dwContentFlags; + + // Hints as for which open method is suitable + DWORD bFileAvailable:1; // If true the file is available locally + DWORD bCanOpenByName:1; + DWORD bCanOpenByDataId:1; + DWORD bCanOpenByCKey:1; + DWORD bCanOpenByEKey:1; + CASC_NAME_TYPE NameType; }; /** @@ -44,7 +76,7 @@ class StorageExplorer { // TODO: CascGetStorageInfo - std::vector enumerateFiles(); + bool enumerateFiles(std::vector& searchResults); /** * @brief extract data of given file to location specified under filesystem diff --git a/src/storage.cc b/src/storage.cc index abe1c15..7d8336f 100644 --- a/src/storage.cc +++ b/src/storage.cc @@ -1,4 +1,3 @@ -#include #include "storage.hpp" int StorageExplorer::openStorage(std::string src) @@ -20,23 +19,30 @@ bool StorageExplorer::closeStorage() return CascCloseStorage(m_hStorage); } -std::vector StorageExplorer::enumerateFiles() +bool StorageExplorer::enumerateFiles(std::vector& searchResults) { - std::vector ret; - CASC_FIND_DATA findData; HANDLE handle = CascFindFirstFile(m_hStorage, "*", &findData, NULL); + if (handle == INVALID_HANDLE_VALUE) { + PLOG_FATAL << "CascFindFirstFile E(" << GetLastError() << ")"; + return false; + } + do { - findData.dwFileSize; - findData.FileKey; - findData.szFileName; - ret.push_back(findData.szFileName); + if (!findData.bFileAvailable) continue; + + STORAGE_SEARCH_RESULT *record = new STORAGE_SEARCH_RESULT(); + record->filename = findData.szFileName; + memcpy(record->CKey, findData.CKey, sizeof(record->CKey)); + memcpy(record->EKey, findData.EKey, sizeof(record->EKey)); + record->dwFileSize = findData.dwFileSize; + searchResults.push_back(record); } while (CascFindNextFile(handle, &findData)); CascFindClose(handle); - return ret; + return true; } size_t StorageExplorer::extractFileToPath(const std::string& storedFilename, const std::string& targetFilename) diff --git a/src/stormex.cc b/src/stormex.cc index 5f42dff..596889d 100644 --- a/src/stormex.cc +++ b/src/stormex.cc @@ -5,11 +5,13 @@ #include #include #include +#include #include "cxxopts.hpp" #include "common.hpp" #include "util.hpp" #include "storage.hpp" +#include "common/Common.h" class StormexContext { public: @@ -31,7 +33,8 @@ class StormexContext { } m_filters; struct { - std::vector extractFiles; + bool doExtractAll; + std::vector xFilenames; std::string outDir; bool stdOut; bool progress; @@ -83,7 +86,7 @@ void parseArguments(int argc, char* argv[]) "\n" "Command-line application to enumerate and extract files from CASC (Content Addressable Storage Container) used in Blizzard games.\n" "\n" - "Expected regex pattern should follow ECMAScript syntax\n"); + "Regex pattern is expected to follow ECMAScript syntax\n"); options .positional_help("[STORAGE]") .show_positional_help(); @@ -91,14 +94,14 @@ void parseArguments(int argc, char* argv[]) options.add_options("Common") ("h,help", "Print help.") ("v,verbose", "Verbose output.", cxxopts::value()) - ("q,quiet", "Supresses output of messages entirely.", cxxopts::value()) + ("q,quiet", "Supresses output entirely.", cxxopts::value()) ("version", "Print version."); options.add_options("Base") - ("S,storage", "Path to directory with CASC.", cxxopts::value(appCtx.m_base.storageSrc), "[PATH]") - ("L,listfile", - "Map filenames from provided newline delimeted (LF or CRLF) textfile, instead of enumerating content of the archive, " - "which is an extensive operation. It combines well when extracting single files, or a small group that matches given substring or regex pattern.", cxxopts::value(appCtx.m_base.listfileSrc), "[FILE]"); + ("S,storage", "Path to directory with CASC.", cxxopts::value(appCtx.m_base.storageSrc), "[PATH]"); + // ("L,listfile", + // "Map filenames from provided newline delimeted (LF or CRLF) textfile, instead of enumerating content of the archive, " + // "which is an extensive operation. It combines well when extracting single files, or a small group that matches given substring or regex pattern.", cxxopts::value(appCtx.m_base.listfileSrc), "[FILE]"); options.add_options("List") ("l,list", "List files inside CASC.", cxxopts::value(appCtx.m_list.listFiles)) @@ -115,10 +118,13 @@ void parseArguments(int argc, char* argv[]) ("E,ex-iregex", "Exclude files matching regex case insensitively.", cxxopts::value>(), "[PATTERN...]"); options.add_options("Extract") - ("x,extract", - "Extract file(s) matching exactly. Argument is optional - if ommitted it will extract all files matching search filters.", - cxxopts::value>(appCtx.m_extract.extractFiles), "[FILE...]") - ("O,outdir", "Output directory for extracted files.", cxxopts::value(appCtx.m_extract.outDir)->default_value("./"), "[PATH]") + ("x,extract-all", + "Extract all files matching search filters.", + cxxopts::value(appCtx.m_extract.doExtractAll)) + ("X,extract-file", + "Extract file(s) matching exactly.", + cxxopts::value>(appCtx.m_extract.xFilenames), "[FILE...]") + ("o,outdir", "Output directory for extracted files.", cxxopts::value(appCtx.m_extract.outDir)->default_value("./"), "[PATH]") ("p,stdout", "Pipe content of a file(s) to stdout instead writing it to the filesystem.", cxxopts::value(appCtx.m_extract.stdOut)) ("P,progress", "Notify about progress during extraction.", cxxopts::value(appCtx.m_extract.progress)) ("n,dry-run", "Simulate extraction process without writing any data to the filesystem.", cxxopts::value(appCtx.m_extract.dryRun)); @@ -133,7 +139,7 @@ void parseArguments(int argc, char* argv[]) } if (result.count("version")) { - std::cerr << "stormex v" << stormexVersion << std::endl; + std::cerr << "stormex v" << stormexVersion << " | CascLib v" << CASCLIB_VERSION_STRING << std::endl; exit(0); } @@ -165,7 +171,7 @@ void parseArguments(int argc, char* argv[]) } } -void extractFiles(StorageExplorer& stExplorer, const std::vector& filesToExtract) +void extractFilenames(StorageExplorer& stExplorer, const std::vector& filesToExtract) { PLOG_DEBUG << "Preparing to extract " << filesToExtract.size() << " files.."; if (appCtx.m_extract.dryRun) { @@ -193,11 +199,18 @@ void extractFiles(StorageExplorer& stExplorer, const std::vector& f } targetFile += storedFilename; - PLOG_DEBUG << "Extracting file: " << storedFilename; + if (appCtx.m_extract.progress) { + // TODO: display progress + } + + PLOG_INFO << "Extracting file " << storedFilename; size_t fileSize = 0; if (!appCtx.m_extract.dryRun) { + // normalize slashes in the paths received from CASC and force '/' + std::replace(targetFile.begin(), targetFile.end(), '\\', PATH_SEP_CHAR); + fileSize = stExplorer.extractFileToPath(storedFilename, targetFile); - PLOG_DEBUG << "Saved at: " << targetFile << " [" << formatFileSize(fileSize) << "]"; + PLOG_DEBUG << "Written " << formatFileSize(fileSize) << " to " << targetFile; } else { } @@ -205,7 +218,7 @@ void extractFiles(StorageExplorer& stExplorer, const std::vector& f } } -bool searchRegexMulti(const std::string filename, const std::vector& patterns) +bool searchRegexMulti(const std::string& filename, const std::vector& patterns) { for (const auto& current : patterns) { if (regex_search(filename, current)) { @@ -216,22 +229,22 @@ bool searchRegexMulti(const std::string filename, const std::vector& return false; } -std::vector filterFiles(const std::vector& inputList) +std::vector filterFiles(const std::vector& inputList) { - std::vector filteredList; + std::vector filteredList; for (const auto& entry : inputList) { if (appCtx.m_filters.searchPhrase.size()) { bool c = false; for (const auto& needle : appCtx.m_filters.searchPhrase) { - c = findStringIC(entry, needle); + c = findStringIC(entry->filename, needle); if (c) break; } if (!c) continue; } - if (appCtx.m_filters.includePatterns.size() && !searchRegexMulti(entry, appCtx.m_filters.includePatterns)) continue; - if (appCtx.m_filters.excludePatterns.size() && searchRegexMulti(entry, appCtx.m_filters.excludePatterns)) continue; + if (appCtx.m_filters.includePatterns.size() && !searchRegexMulti(entry->filename, appCtx.m_filters.includePatterns)) continue; + if (appCtx.m_filters.excludePatterns.size() && searchRegexMulti(entry->filename, appCtx.m_filters.excludePatterns)) continue; filteredList.push_back(entry); } @@ -253,10 +266,13 @@ std::vector readListFile(const std::string& filename) return filelist; } -void listFiles(StorageExplorer& stExplorer) +std::vector enumerateFiles(StorageExplorer& stExplorer) { PLOG_INFO << "Enumerating all files in storage.."; - auto inputList = stExplorer.enumerateFiles(); + std::vector inputList; + if (!stExplorer.enumerateFiles(inputList)) { + return inputList; + } auto filteredList = inputList; if (appCtx.m_filters.searchPhrase.size() || appCtx.m_filters.includePatterns.size() || appCtx.m_filters.excludePatterns.size()) { @@ -264,11 +280,8 @@ void listFiles(StorageExplorer& stExplorer) filteredList = filterFiles(inputList); } - for (const auto& entry : filteredList) { - std::cout << entry << std::endl; - } - - PLOG_DEBUG << "count " << inputList.size() << " : " << filteredList.size(); + PLOG_DEBUG << "list count " << inputList.size() << " : " << filteredList.size(); + return filteredList; } int main(int argc, char* argv[]) @@ -286,11 +299,35 @@ int main(int argc, char* argv[]) PLOG_INFO << "Storage opened " << static_cast(stExplorer.getHandle()); try { + auto fResults = enumerateFiles(stExplorer); + if (appCtx.m_list.listFiles) { - listFiles(stExplorer); + for (const auto& entry : fResults) { + char keyBuff[MD5_STRING_SIZE + 1]; + std::string tmps; + if (appCtx.m_list.showDetails) { + tmps = formatFileSize(entry->dwFileSize); + std::cout << tmps << std::setw(8 - tmps.length()) << " "; + std::cout << StringFromMD5((LPBYTE)entry->CKey, keyBuff) << " "; + std::cout << StringFromMD5((LPBYTE)entry->EKey, keyBuff) << " "; + } + std::cout << entry->filename; + std::cout << std::endl; + } } - else if (appCtx.m_extract.extractFiles.size()) { - extractFiles(stExplorer, appCtx.m_extract.extractFiles); + else if (appCtx.m_extract.doExtractAll) { + std::vector fList; + for (const auto& item : fResults) { + fList.push_back(item->filename); + } + extractFilenames(stExplorer, fList); + } + else if (appCtx.m_extract.xFilenames.size()) { + for (auto& item : appCtx.m_extract.xFilenames) { + // force backslashes regardless of the platform - that's the expected output from CASC anyway, and it'll get normalized later + std::replace(item.begin(), item.end(), '/', '\\'); + } + extractFilenames(stExplorer, appCtx.m_extract.xFilenames); } } catch (const std::exception& e) { stExplorer.closeStorage(); diff --git a/src/util.cc b/src/util.cc index 173db32..648e79c 100644 --- a/src/util.cc +++ b/src/util.cc @@ -1,4 +1,6 @@ #include +#include +#include #include #include #include @@ -22,9 +24,6 @@ bool pathExists(const std::string& target) int ensureDirExists(std::string strDestName) { - // normalize slashes in the paths received from CASC and force '/' - std::replace(strDestName.begin(), strDestName.end(), '\\', '/'); - // ensure directory path to the file exists size_t pos = -1; while ((pos = strDestName.find('/', pos + 1)) != std::string::npos) { @@ -52,21 +51,21 @@ template std::string valueToString(T num) { std::ostringstream convert; - convert << num; + convert << std::setprecision(8) << num; return convert.str(); } static double roundOff(double n) { - double d = n * 100.0; + double d = n * 10.0; int i = d + 0.5; - d = (float)i / 100.0; + d = (float)i / 10.0; return d; } std::string formatFileSize(size_t size) { - static const char *SIZES[] = { "B", "KB", "MB", "GB" }; + static const char *SIZES[] = { "B", "K", "M", "G" }; int div = 0; size_t rem = 0; @@ -77,7 +76,9 @@ std::string formatFileSize(size_t size) } double size_d = (float)size + (float)rem / 1024.0; - std::string result = valueToString(roundOff(size_d)) + " " + SIZES[div]; + std::string result = valueToString(roundOff(size_d)) + SIZES[div]; + std::replace(result.begin(), result.end(), '.', ','); + return result; }