From a96a30154d06b30f8e623271aab9d5555726a55f Mon Sep 17 00:00:00 2001 From: Jussi Kansanen Date: Mon, 9 Jun 2014 16:43:58 +0300 Subject: [PATCH 01/16] Add flags for OpenMP. --- Makefile.am | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Makefile.am b/Makefile.am index d6bc12d..c5dc65d 100755 --- a/Makefile.am +++ b/Makefile.am @@ -47,7 +47,7 @@ verificationpacket.cpp verificationpacket.h LDADD = -lstdc++ -AM_CXXFLAGS = -Wall +AM_CXXFLAGS = -Wall $(OPENMP_CXXFLAGS) EXTRA_DIST = PORTING ROADMAP par2cmdline.sln par2cmdline.vcproj \ par2.1 \ From 442eea6b927d6b8714d2fb1a4d33a9de70e9dab8 Mon Sep 17 00:00:00 2001 From: Jussi Kansanen Date: Mon, 9 Jun 2014 17:07:29 +0300 Subject: [PATCH 02/16] Add check for OpenMP. --- configure.ac | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/configure.ac b/configure.ac index 6d36d10..3456fca 100755 --- a/configure.ac +++ b/configure.ac @@ -35,7 +37,8 @@ dnl Checks for programs. AC_PROG_CXX AC_PROG_INSTALL -dnl Checks for libraries. +dnl Set language to C++ +AC_LANG(C++) dnl Checks for header files. AC_HEADER_DIRENT @@ -52,6 +55,9 @@ AC_C_INLINE AC_SYS_LARGEFILE AC_FUNC_FSEEKO +dnl Check for OpenMP +AC_OPENMP + dnl Checks for library functions. AC_FUNC_MEMCMP AC_CHECK_FUNCS([stricmp] [strcasecmp]) From 262288bdc6e6e9cea8384701646b2c2f3269ee27 Mon Sep 17 00:00:00 2001 From: Jussi Kansanen Date: Mon, 9 Jun 2014 17:10:56 +0300 Subject: [PATCH 03/16] ... --- AUTHORS | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/AUTHORS b/AUTHORS index 74df19d..a50c1ae 100644 --- a/AUTHORS +++ b/AUTHORS @@ -1,3 +1,9 @@ +par2cmdline-mt: + +Jussi Kansanen + +par2cmdline: + Peter Brian Clements Marcel Partap Ike Devolder From c05a96d459a045f0f43a7e6aa29388acf7fabb1c Mon Sep 17 00:00:00 2001 From: Jussi Kansanen Date: Mon, 9 Jun 2014 17:20:10 +0300 Subject: [PATCH 04/16] Parallelize creator with OpenMP. --- par2creator.cpp | 2 ++ 1 file changed, 2 insertions(+) diff --git a/par2creator.cpp b/par2creator.cpp index fa8fcc8..baed047 100644 --- a/par2creator.cpp +++ b/par2creator.cpp @@ -924,6 +924,7 @@ bool Par2Creator::ProcessData(u64 blockoffset, size_t blocklength) } // For each output block + #pragma omp parallel for for (u32 outputblock=0; outputblock Date: Mon, 9 Jun 2014 17:25:24 +0300 Subject: [PATCH 05/16] Parallelize repair with OpenMP. --- par2repairer.cpp | 2 ++ 1 file changed, 2 insertions(+) diff --git a/par2repairer.cpp b/par2repairer.cpp index d3d3cf3..15ca316 100644 --- a/par2repairer.cpp +++ b/par2repairer.cpp @@ -2198,6 +2198,7 @@ bool Par2Repairer::ProcessData(u64 blockoffset, size_t blocklength) } // For each output block + #pragma omp parallel for for (u32 outputindex=0; outputindex Date: Mon, 9 Jun 2014 17:28:50 +0300 Subject: [PATCH 06/16] Include omp.h for OpenMP --- par2cmdline.h | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/par2cmdline.h b/par2cmdline.h index d3ba66d..bcbb534 100644 --- a/par2cmdline.h +++ b/par2cmdline.h @@ -300,5 +300,11 @@ using namespace std; #define DEBUG_NEW new(_NORMAL_BLOCK, THIS_FILE, __LINE__) #endif +// OpenMP +#ifdef _OPENMP +# include +#endif + + #endif // __PARCMDLINE_H__ From 312d097088cf3abdd5f36cd4b54ccd5b54372e85 Mon Sep 17 00:00:00 2001 From: Jussi Kansanen Date: Mon, 9 Jun 2014 18:07:58 +0300 Subject: [PATCH 07/16] Add a command line option for settings number of threads. --- commandline.cpp | 34 ++++++++++++++++++++++++++++++++-- 1 file changed, 32 insertions(+), 2 deletions(-) diff --git a/commandline.cpp b/commandline.cpp index 3136003..f8cc559 100644 --- a/commandline.cpp +++ b/commandline.cpp @@ -91,6 +91,7 @@ cout << "Copyright (C) 2003-2015 Peter Brian Clements." << endl << "Copyright (C) 2011-2012 Marcel Partap." << endl << "Copyright (C) 2012-2015 Ike Devolder." << endl + << "Copyright (C) 2014 Jussi Kansanen." << endl << endl << "par2cmdline comes with ABSOLUTELY NO WARRANTY." << endl << endl @@ -118,7 +119,12 @@ void CommandLine::usage(void) "\n" " -a : Set the main par2 archive name\n" " -b : Set the Block-Count\n" - " -s : Set the Block-Size (Don't use both -b and -s)\n" + " -s : Set the Block-Size (Don't use both -b and -s)\n"; +#ifdef _OPENMP + cout << + " -t : Number of threads to use (" << omp_get_max_threads() << " detected)\n"; +#endif + cout << " -r : Level of Redundancy (%%)\n" " -r : Redundancy target size, =g(iga),m(ega),k(ilo) bytes\n" " -c : Recovery block count (Don't use both -r and -c)\n" @@ -336,6 +342,30 @@ bool CommandLine::Parse(int argc, char *argv[]) } } break; + + case 't': // Set the amount of threads + { + u32 nthreads = 0; + char *p = &argv[0][2]; + + while (*p && isdigit(*p)) + { + nthreads = nthreads * 10 + (*p - '0'); + p++; + } + + if (!nthreads) + { + cerr << "Invalid thread option: " << argv[0] << endl; + return false; + } + + // Sets the number of threads to use + omp_set_num_threads(nthreads); + + } + break; + case 'r': // Set the amount of redundancy required { From 157d739a55ed18a4070a5090a154207d47657299 Mon Sep 17 00:00:00 2001 From: Jussi Kansanen Date: Mon, 9 Jun 2014 19:03:09 +0300 Subject: [PATCH 08/16] Update man page. --- par2.1 | 13 ++++++++++--- 1 file changed, 10 insertions(+), 3 deletions(-) diff --git a/par2.1 b/par2.1 index 019220f..5c45b25 100644 --- a/par2.1 +++ b/par2.1 @@ -51,6 +51,9 @@ Set the Block\(hyCount .B \-s .RB "Set the Block\(hySize (don't use both " "\-b" " and " "\-s" ")" .TP +.B \-t +.RB "Number of threads to use (Auto-detected)" +.TP .B \-r Level of redundancy (percentage) .TP @@ -95,3 +98,7 @@ Peter Brian Clements Marcel Partap .br Ike Devolder +.br +par2cmdline-mt: Jussi Kansanen +.br + From a5d828620135a17e960420f71a575fd27cf9d14b Mon Sep 17 00:00:00 2001 From: Jussi Kansanen Date: Mon, 9 Jun 2014 19:12:37 +0300 Subject: [PATCH 09/16] Enable OpenMP. --- par2cmdline.vcproj | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) From 39edd3264bc394254692f81b9922601f532264f1 Mon Sep 17 00:00:00 2001 From: Jussi Kansanen Date: Mon, 9 Jun 2014 19:18:32 +0300 Subject: [PATCH 10/16] Update READMEs. --- README | 8 ++++++-- README.md | 11 +++++++---- 2 files changed, 13 insertions(+), 6 deletions(-) diff --git a/README b/README index a8dd396..0b1eec6 100644 --- a/README +++ b/README @@ -1,6 +1,9 @@ -par2cmdline is a PAR 2.0 compatible file verification and repair tool. +par2cmdline-mt is a multithreaded (OpenMP) PAR 2.0 compatible file verification and repair tool. To see the ongoing development see +https://github.com/jkansanen/par2cmdline-mt + +Par2cmdline was forked from https://github.com/BlackIkeEagle/par2cmdline The original development was done on Sourceforge but stalled. @@ -66,7 +69,7 @@ If you have only downloaded a precompiled executable, then the source code should be available from the same location where you downloaded the executable from. -If you have MS Visual Studio .NET, then just open the par2cmdline.sln +*NOT TESTED WITH PAR2CMDLINE-MT* If you have MS Visual Studio .NET, then just open the par2cmdline.sln file and compile. You should then copy par2cmdline.exe to an appropriate location that is on your path. @@ -106,6 +109,7 @@ The command line parameters for par2cmdline are as follow: required on create, optional for verify and repair -b : Set the Block-Count -s : Set the Block-Size (Don't use both -b and -s) + -t : Number of threads to use (Auto-detected) -r : Level of Redundancy (%) -r : Redundancy target size, =g(iga),m(ega),k(ilo) bytes -c : Recovery block count (don't use both -r and -c) diff --git a/README.md b/README.md index e1acf66..b9770f2 100644 --- a/README.md +++ b/README.md @@ -1,8 +1,10 @@ -#par2cmdline +#par2cmdline-mt -**par2cmdline** is a PAR 2.0 compatible file verification and repair tool. +**par2cmdline-mt** is a multithreaded (OpenMP) PAR 2.0 compatible file verification and repair tool. -To see the ongoing development see +To see the ongoing development see + +Par2cmdline was forked from The original development was done on Sourceforge but stalled. @@ -46,7 +48,7 @@ You should have received par2cmdline in the form of source code which you can co If you have only downloaded a precompiled executable, then the source code should be available from the same location where you downloaded the executable from. -If you have MS Visual Studio .NET, then just open the *par2cmdline.sln* file and compile. You should then copy *par2cmdline.exe* to an appropriate location that is on your path. +*NOT TESTED WITH PAR2CMDLINE-MT* If you have MS Visual Studio .NET, then just open the *par2cmdline.sln* file and compile. You should then copy *par2cmdline.exe* to an appropriate location that is on your path. To compile on Linux and other Unix variants use the following commands: @@ -84,6 +86,7 @@ The command line parameters for par2cmdline are as follow: required on create, optional for verify and repair -b : Set the Block-Count -s : Set the Block-Size (Don't use both -b and -s) + -t : Number of threads to use (Auto-detected) -r : Level of Redundancy (%) -r : Redundancy target size, =g(iga),m(ega),k(ilo) bytes -c : Recovery block count (don't use both -r and -c) From 426d72c4ca3f282e2a4d9fd03648ec3614755865 Mon Sep 17 00:00:00 2001 From: Jussi Kansanen Date: Tue, 10 Jun 2014 12:55:54 +0300 Subject: [PATCH 11/16] Update command line options. --- commandline.cpp | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/commandline.cpp b/commandline.cpp index f8cc559..425b47b 100644 --- a/commandline.cpp +++ b/commandline.cpp @@ -119,12 +119,7 @@ void CommandLine::usage(void) "\n" " -a : Set the main par2 archive name\n" " -b : Set the Block-Count\n" - " -s : Set the Block-Size (Don't use both -b and -s)\n"; -#ifdef _OPENMP - cout << - " -t : Number of threads to use (" << omp_get_max_threads() << " detected)\n"; -#endif - cout << + " -s : Set the Block-Size (Don't use both -b and -s)\n" " -r : Level of Redundancy (%%)\n" " -r : Redundancy target size, =g(iga),m(ega),k(ilo) bytes\n" " -c : Recovery block count (Don't use both -r and -c)\n" @@ -132,7 +127,12 @@ void CommandLine::usage(void) " -u : Uniform recovery file sizes\n" " -l : Limit size of recovery files (Don't use both -u and -l)\n" " -n : Number of recovery files (Don't use both -n and -l)\n" - " -m : Memory (in MB) to use\n" + " -m : Memory (in MB) to use\n"; +#ifdef _OPENMP + cout << + " -t : Number of threads to use (" << omp_get_max_threads() << " detected)\n"; +#endif + cout << " -v [-v] : Be more verbose\n" " -q [-q] : Be more quiet (-q -q gives silence)\n" " -p : Purge backup files and par files on successful recovery or\n" From 307c992d86b95033cfb823bc6961c3e413b39042 Mon Sep 17 00:00:00 2001 From: Jussi Kansanen Date: Tue, 10 Jun 2014 12:58:37 +0300 Subject: [PATCH 12/16] Update READMEs. --- README | 2 +- README.md | 3 ++- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/README b/README index 0b1eec6..0ce51d2 100644 --- a/README +++ b/README @@ -109,7 +109,6 @@ The command line parameters for par2cmdline are as follow: required on create, optional for verify and repair -b : Set the Block-Count -s : Set the Block-Size (Don't use both -b and -s) - -t : Number of threads to use (Auto-detected) -r : Level of Redundancy (%) -r : Redundancy target size, =g(iga),m(ega),k(ilo) bytes -c : Recovery block count (don't use both -r and -c) @@ -118,6 +117,7 @@ The command line parameters for par2cmdline are as follow: -l : Limit size of recovery files (Don't use both -u and -l) -n : Number of recovery files (Don't use both -n and -l) -m : Memory (in MB) to use + -t : Number of threads to use (Auto-detected) -v [-v] : Be more verbose -q [-q] : Be more quiet (-qq gives silence) -p : Purge backup files and par files on successful recovery or diff --git a/README.md b/README.md index b9770f2..248009b 100644 --- a/README.md +++ b/README.md @@ -86,7 +86,7 @@ The command line parameters for par2cmdline are as follow: required on create, optional for verify and repair -b : Set the Block-Count -s : Set the Block-Size (Don't use both -b and -s) - -t : Number of threads to use (Auto-detected) + -r : Level of Redundancy (%) -r : Redundancy target size, =g(iga),m(ega),k(ilo) bytes -c : Recovery block count (don't use both -r and -c) @@ -95,6 +95,7 @@ The command line parameters for par2cmdline are as follow: -l : Limit size of recovery files (Don't use both -u and -l) -n : Number of recovery files (Don't use both -n and -l) -m : Memory (in MB) to use + -t : Number of threads to use (Auto-detected) -v [-v] : Be more verbose -q [-q] : Be more quiet (-qq gives silence) -p : Purge backup files and par files on successful recovery or From 1bf8a78702d5e231d8ab40fee2e263bee7aaa0ad Mon Sep 17 00:00:00 2001 From: Jussi Kansanen Date: Tue, 10 Jun 2014 13:08:52 +0300 Subject: [PATCH 13/16] Update man page. --- par2.1 | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/par2.1 b/par2.1 index 5c45b25..b6e912a 100644 --- a/par2.1 +++ b/par2.1 @@ -50,9 +50,7 @@ Set the Block\(hyCount .TP .B \-s .RB "Set the Block\(hySize (don't use both " "\-b" " and " "\-s" ")" -.TP -.B \-t -.RB "Number of threads to use (Auto-detected)" + .TP .B \-r Level of redundancy (percentage) @@ -78,6 +76,9 @@ Uniform recovery file sizes .B \-m Memory (in MB) to use .TP +.B \-t +.RB "Number of threads to use (Auto-detected)" +.TP .B \-v [\-v] Be more verbose .TP From eaf83a4c0f834ad7f9da07bbb08c9a7d0b2a1181 Mon Sep 17 00:00:00 2001 From: Jussi Kansanen Date: Tue, 10 Jun 2014 13:13:18 +0300 Subject: [PATCH 14/16] Update README.md --- README.md | 1 - 1 file changed, 1 deletion(-) diff --git a/README.md b/README.md index 248009b..8417693 100644 --- a/README.md +++ b/README.md @@ -86,7 +86,6 @@ The command line parameters for par2cmdline are as follow: required on create, optional for verify and repair -b : Set the Block-Count -s : Set the Block-Size (Don't use both -b and -s) - -r : Level of Redundancy (%) -r : Redundancy target size, =g(iga),m(ega),k(ilo) bytes -c : Recovery block count (don't use both -r and -c) From 0e5c05dc6641611b4a0dc2deaa42d28da8f4ee13 Mon Sep 17 00:00:00 2001 From: Jussi Kansanen Date: Wed, 8 Apr 2015 20:14:43 +0300 Subject: [PATCH 15/16] Dont set number of threads if OpenMP isn't used (reported by Yutaka Sawada). --- commandline.cpp | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/commandline.cpp b/commandline.cpp index 073947f..1d47072 100644 --- a/commandline.cpp +++ b/commandline.cpp @@ -129,10 +129,10 @@ void CommandLine::usage(void) " -n : Number of recovery files (Don't use both -n and -l)\n" " -m : Memory (in MB) to use\n"; #ifdef _OPENMP - cout << + cout << " -t : Number of threads to use (" << omp_get_max_threads() << " detected)\n"; #endif - cout << + cout << " -v [-v] : Be more verbose\n" " -q [-q] : Be more quiet (-q -q gives silence)\n" " -p : Purge backup files and par files on successful recovery or\n" @@ -342,12 +342,13 @@ bool CommandLine::Parse(int argc, char *argv[]) } } break; - + +#ifdef _OPENMP case 't': // Set the amount of threads { u32 nthreads = 0; char *p = &argv[0][2]; - + while (*p && isdigit(*p)) { nthreads = nthreads * 10 + (*p - '0'); @@ -360,13 +361,12 @@ bool CommandLine::Parse(int argc, char *argv[]) return false; } - // Sets the number of threads to use + // Sets the number of threads to use omp_set_num_threads(nthreads); - + } break; - - +#endif case 'r': // Set the amount of redundancy required { if (operation != opCreate) From 131980f0d83f224305bf66d313b6319d09d525ba Mon Sep 17 00:00:00 2001 From: Jussi Kansanen Date: Wed, 5 Aug 2015 23:27:28 +0300 Subject: [PATCH] Fix race condition in progress indicator (reported by Phaneron). --- par2creator.cpp | 1 + par2repairer.cpp | 1 + 2 files changed, 2 insertions(+) diff --git a/par2creator.cpp b/par2creator.cpp index 3679b56..099d9a0 100644 --- a/par2creator.cpp +++ b/par2creator.cpp @@ -937,6 +937,7 @@ bool Par2Creator::ProcessData(u64 blockoffset, size_t blocklength) { // Update a progress indicator u32 oldfraction = (u32)(1000 * progress / totaldata); + #pragma omp atomic progress += blocklength; u32 newfraction = (u32)(1000 * progress / totaldata); diff --git a/par2repairer.cpp b/par2repairer.cpp index 351488d..73c5b77 100644 --- a/par2repairer.cpp +++ b/par2repairer.cpp @@ -2231,6 +2231,7 @@ bool Par2Repairer::ProcessData(u64 blockoffset, size_t blocklength) { // Update a progress indicator u32 oldfraction = (u32)(1000 * progress / totaldata); + #pragma omp atomic progress += blocklength; u32 newfraction = (u32)(1000 * progress / totaldata);