/sys/contrib/zstd/contrib/seekable_format/examples/parallel_processing.c

https://bitbucket.org/freebsd/freebsd-base · C · 194 lines · 139 code · 36 blank · 19 comment · 24 complexity · d9fd227fb7c38320039d6afea31e589d MD5 · raw file

  1. /*
  2. * Copyright (c) 2017-present, Facebook, Inc.
  3. * All rights reserved.
  4. *
  5. * This source code is licensed under both the BSD-style license (found in the
  6. * LICENSE file in the root directory of this source tree) and the GPLv2 (found
  7. * in the COPYING file in the root directory of this source tree).
  8. */
  9. /*
  10. * A simple demo that sums up all the bytes in the file in parallel using
  11. * seekable decompression and the zstd thread pool
  12. */
  13. #include <stdlib.h> // malloc, exit
  14. #include <stdio.h> // fprintf, perror, feof
  15. #include <string.h> // strerror
  16. #include <errno.h> // errno
  17. #define ZSTD_STATIC_LINKING_ONLY
  18. #include <zstd.h> // presumes zstd library is installed
  19. #include <zstd_errors.h>
  20. #if defined(WIN32) || defined(_WIN32)
  21. # include <windows.h>
  22. # define SLEEP(x) Sleep(x)
  23. #else
  24. # include <unistd.h>
  25. # define SLEEP(x) usleep(x * 1000)
  26. #endif
  27. #include "pool.h" // use zstd thread pool for demo
  28. #include "zstd_seekable.h"
  29. #define MIN(a, b) ((a) < (b) ? (a) : (b))
  30. static void* malloc_orDie(size_t size)
  31. {
  32. void* const buff = malloc(size);
  33. if (buff) return buff;
  34. /* error */
  35. perror("malloc");
  36. exit(1);
  37. }
  38. static void* realloc_orDie(void* ptr, size_t size)
  39. {
  40. ptr = realloc(ptr, size);
  41. if (ptr) return ptr;
  42. /* error */
  43. perror("realloc");
  44. exit(1);
  45. }
  46. static FILE* fopen_orDie(const char *filename, const char *instruction)
  47. {
  48. FILE* const inFile = fopen(filename, instruction);
  49. if (inFile) return inFile;
  50. /* error */
  51. perror(filename);
  52. exit(3);
  53. }
  54. static size_t fread_orDie(void* buffer, size_t sizeToRead, FILE* file)
  55. {
  56. size_t const readSize = fread(buffer, 1, sizeToRead, file);
  57. if (readSize == sizeToRead) return readSize; /* good */
  58. if (feof(file)) return readSize; /* good, reached end of file */
  59. /* error */
  60. perror("fread");
  61. exit(4);
  62. }
  63. static size_t fwrite_orDie(const void* buffer, size_t sizeToWrite, FILE* file)
  64. {
  65. size_t const writtenSize = fwrite(buffer, 1, sizeToWrite, file);
  66. if (writtenSize == sizeToWrite) return sizeToWrite; /* good */
  67. /* error */
  68. perror("fwrite");
  69. exit(5);
  70. }
  71. static size_t fclose_orDie(FILE* file)
  72. {
  73. if (!fclose(file)) return 0;
  74. /* error */
  75. perror("fclose");
  76. exit(6);
  77. }
  78. static void fseek_orDie(FILE* file, long int offset, int origin) {
  79. if (!fseek(file, offset, origin)) {
  80. if (!fflush(file)) return;
  81. }
  82. /* error */
  83. perror("fseek");
  84. exit(7);
  85. }
  86. struct sum_job {
  87. const char* fname;
  88. unsigned long long sum;
  89. unsigned frameNb;
  90. int done;
  91. };
  92. static void sumFrame(void* opaque)
  93. {
  94. struct sum_job* job = (struct sum_job*)opaque;
  95. job->done = 0;
  96. FILE* const fin = fopen_orDie(job->fname, "rb");
  97. ZSTD_seekable* const seekable = ZSTD_seekable_create();
  98. if (seekable==NULL) { fprintf(stderr, "ZSTD_seekable_create() error \n"); exit(10); }
  99. size_t const initResult = ZSTD_seekable_initFile(seekable, fin);
  100. if (ZSTD_isError(initResult)) { fprintf(stderr, "ZSTD_seekable_init() error : %s \n", ZSTD_getErrorName(initResult)); exit(11); }
  101. size_t const frameSize = ZSTD_seekable_getFrameDecompressedSize(seekable, job->frameNb);
  102. unsigned char* data = malloc_orDie(frameSize);
  103. size_t result = ZSTD_seekable_decompressFrame(seekable, data, frameSize, job->frameNb);
  104. if (ZSTD_isError(result)) { fprintf(stderr, "ZSTD_seekable_decompressFrame() error : %s \n", ZSTD_getErrorName(result)); exit(12); }
  105. unsigned long long sum = 0;
  106. size_t i;
  107. for (i = 0; i < frameSize; i++) {
  108. sum += data[i];
  109. }
  110. job->sum = sum;
  111. job->done = 1;
  112. fclose(fin);
  113. ZSTD_seekable_free(seekable);
  114. free(data);
  115. }
  116. static void sumFile_orDie(const char* fname, int nbThreads)
  117. {
  118. POOL_ctx* pool = POOL_create(nbThreads, nbThreads);
  119. if (pool == NULL) { fprintf(stderr, "POOL_create() error \n"); exit(9); }
  120. FILE* const fin = fopen_orDie(fname, "rb");
  121. ZSTD_seekable* const seekable = ZSTD_seekable_create();
  122. if (seekable==NULL) { fprintf(stderr, "ZSTD_seekable_create() error \n"); exit(10); }
  123. size_t const initResult = ZSTD_seekable_initFile(seekable, fin);
  124. if (ZSTD_isError(initResult)) { fprintf(stderr, "ZSTD_seekable_init() error : %s \n", ZSTD_getErrorName(initResult)); exit(11); }
  125. unsigned const numFrames = ZSTD_seekable_getNumFrames(seekable);
  126. struct sum_job* jobs = (struct sum_job*)malloc(numFrames * sizeof(struct sum_job));
  127. unsigned fnb;
  128. for (fnb = 0; fnb < numFrames; fnb++) {
  129. jobs[fnb] = (struct sum_job){ fname, 0, fnb, 0 };
  130. POOL_add(pool, sumFrame, &jobs[fnb]);
  131. }
  132. unsigned long long total = 0;
  133. for (fnb = 0; fnb < numFrames; fnb++) {
  134. while (!jobs[fnb].done) SLEEP(5); /* wake up every 5 milliseconds to check */
  135. total += jobs[fnb].sum;
  136. }
  137. printf("Sum: %llu\n", total);
  138. POOL_free(pool);
  139. ZSTD_seekable_free(seekable);
  140. fclose(fin);
  141. free(jobs);
  142. }
  143. int main(int argc, const char** argv)
  144. {
  145. const char* const exeName = argv[0];
  146. if (argc!=3) {
  147. fprintf(stderr, "wrong arguments\n");
  148. fprintf(stderr, "usage:\n");
  149. fprintf(stderr, "%s FILE NB_THREADS\n", exeName);
  150. return 1;
  151. }
  152. {
  153. const char* const inFilename = argv[1];
  154. int const nbThreads = atoi(argv[2]);
  155. sumFile_orDie(inFilename, nbThreads);
  156. }
  157. return 0;
  158. }