drupalorg-git /shared.php

Language PHP Lines 537
MD5 Hash 115c81228a4bfb0a2764fada121a2fab
Repository https://github.com/mrsinguyen/drupalorg-git.git View Raw File View Project SPDX
  1
  2
  3
  4
  5
  6
  7
  8
  9
 10
 11
 12
 13
 14
 15
 16
 17
 18
 19
 20
 21
 22
 23
 24
 25
 26
 27
 28
 29
 30
 31
 32
 33
 34
 35
 36
 37
 38
 39
 40
 41
 42
 43
 44
 45
 46
 47
 48
 49
 50
 51
 52
 53
 54
 55
 56
 57
 58
 59
 60
 61
 62
 63
 64
 65
 66
 67
 68
 69
 70
 71
 72
 73
 74
 75
 76
 77
 78
 79
 80
 81
 82
 83
 84
 85
 86
 87
 88
 89
 90
 91
 92
 93
 94
 95
 96
 97
 98
 99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
<?php

if (!defined('LOGLEVEL')) {
  // Let an environment variable set the log level
  $level = getenv('LOGLEVEL');
  if (is_string($level)) {
    define('LOGLEVEL', (int) $level);
  }
  else {
    // Or default to 'normal'
    define('LOGLEVEL', 3);
  }
}

if (!defined('CVS2GIT')) {
  $c2g = getenv('CVS2GIT');
  if (is_string($c2g)) {
    define('CVS2GIT', $c2g);
  }
  else {
    // Or default to 'cvs2git', so whatever's on the PATH
    define('CVS2GIT', 'cvs2git');
  }
}

global $rename_patterns;

$rename_patterns = array(
  'core' => array(
    'branches' => array(
      // Strip DRUPAL- prefix.
      '/^DRUPAL-/' => '',
      // One version for 4-7 and prior...
      '/^(\d)-(\d)$/' => '\1.\2.x',
      // And another for D5 and later
      '/^(\d)$/' => '\1.x',
      // Plus one for the weird, weird security syntax
      '/^(\d)-(\d+)-SECURITY$/' => '\1.x-\2-security',
    ),
    'tags' => array(
      // Strip DRUPAL- prefix.
      '/^DRUPAL-/' => '',
      // 4-7 and earlier base transform
      '/^(\d)-(\d)-(\d+)/' => '\1.\2.\3',
      // 5 and later base transform
      '/^(\d)-(\d+)/' => '\1.\2',
      // And now lowercase all possible extra strings
      '/UNSTABLE/' => 'unstable',
      '/ALPHA/' => 'alpha',
      '/BETA/' => 'beta',
      '/RC/' => 'rc',
    ),
    'tagmatch' => '/^DRUPAL-\d(-\d)?-\d+(-(\w+)(-)?(\d+)?)?$/',
  ),
  'contrib' => array(
    'branches' => array(
      // Strip DRUPAL- prefix.
      '/^DRUPAL-/' => '',
      // Ensure that any "pseudo" branch names are made to follow the official pattern
      '/^(\d(-\d)?)$/' => '\1--1',
      // With pseudonames converted, do full transform. One version for 4-7 and prior...
      '/^(\d)-(\d)--(\d+)$/' => '\1.\2.x-\3.x',
      // And another for D5 and later
      '/^(\d)--(\d+)$/' => '\1.x-\2.x',
      // Plus one for the weird, weird security syntax
      '/^(\d)--(\d+)-(\d+)-SECURITY$/' => '\1.x-\2.\3-security',
    ),
    'tags' => array(
      // Strip DRUPAL- prefix.
      '/^DRUPAL-/' => '',
      // 4-7 and earlier base transform
      '/^(\d)-(\d)--(\d+)-(\d+)$/' => '\1.\2.x-\3.\4',
      // Aggressively normalize post-version strings for 4-7 and earlier
      '/^(\d)-(\d)--(\d+)-(\d+)(-)?([A-Za-z]+)(-)?(\d+)?$/' => '\1.\2.x-\3.\4-\6\8',
      // 5 and later base transform
      '/^(\d)--(\d+)-(\d+)$/' => '\1.x-\2.\3',
      // Aggressively normalize post-version strings for 5 and later
      '/^(\d)--(\d+)-(\d+)(-)?([A-Za-z]+)(-)?(\d+)?$/' => '\1.x-\2.\3-\5\7',
      // And now lowercase all possible extra strings
      '/UNSTABLE/' => 'unstable',
      '/ALPHA/' => 'alpha',
      '/BETA/' => 'beta',
      '/RC/' => 'rc',
    ),
    'tagmatch' => '/^DRUPAL-\d(-\d)?--\d+-\d+((-)?(\w+)(-)?(\d+)?)?$/',
  ),
);

function git_invoke($command, $fail_safe = FALSE, $repository_path = NULL, $cwd = NULL, $env = NULL) {
  if (!isset($env)) {
    $env = $_ENV;
  }
  if ($repository_path) {
    $env['GIT_DIR'] = $repository_path;
  }

  $descriptor_spec = array(
    1 => array('pipe', 'w'),
    2 => array('pipe', 'w'),
  );

  git_log('Invoking ' . $command, 'DEBUG');

  $process = proc_open($command, $descriptor_spec, $pipes, $cwd, $env);
  if (is_resource($process)) {
    $stdout = stream_get_contents($pipes[1]);
    fclose($pipes[1]);
    $stderr = stream_get_contents($pipes[2]);
    fclose($pipes[2]);

    $return_code = proc_close($process);

    if ($return_code != 0 && !$fail_safe) {
      throw new Exception("Invocation of '" . $command . "' failed with return code " . $return_code .": \n" . $stdout . $stderr);
    }

    return $stdout;
  }
}

function is_empty_dir($dir){
  $files = @scandir($dir);
  return (!$files || count($files) <= 2);
}

/**
 * Check if directory has any CVS information.
 *
 * This is actually sort of a recursive problem. If any subdirectory has
 * CVS information it can be imported.
 */
function is_cvs_dir($dir) {
  $files = @scandir($dir);

  // If there are no files, fail early.
  if (!$files) {
    return FALSE;
  }

  foreach ($files as $file) {
    $absolute = $dir . '/' . $file;

    // Skip POSIX aliases
    if ($file == '.' || $file == '..') continue;

    if (is_dir($absolute) && $file == 'Attic') {
      return TRUE;
    }
    elseif (strpos($file, ',v') !== FALSE) {
      return TRUE;
    }
    elseif (is_dir($absolute) && is_cvs_dir($absolute)) {
      return TRUE;
    }
  }
  return FALSE;
}

/**
 * Recursively delete a directory on a local filesystem.
 *
 * @param string $path
 *   The path to the directory.
 */
function rmdirr($path) {
  foreach (new RecursiveIteratorIterator(new RecursiveDirectoryIterator($path), RecursiveIteratorIterator::CHILD_FIRST) as $item) {
    $item->isFile() ? unlink($item) : rmdir($item);
  }
  rmdir($path);
}

function git_log($message, $level = 'NORMAL', $project = NULL) {
  $loglevels = array(
    'WARN' => 1,
    'QUIET' => 2,
    'NORMAL' => 3,
    'INFO' => 4,
    'DEBUG' => 5,
  );
  if (LOGLEVEL !== 0 && LOGLEVEL >= $loglevels[$level]) {
    if (isset($project)) {
      echo "[" . date('Y-m-d H:i:s') . "] [$level] [$project] $message\n";
    }
    else {
      echo "[" . date('Y-m-d H:i:s') . "] [$level] $message\n";
    }
  }
}

/**
 * Helper function to import a directory to a git repository.
 */
function import_directory($config, $root, $source, $destination, $wipe = FALSE) {
  global $rename_patterns, $wd;
  // Ensure no trailing slashes for cleanliness
  $source = rtrim($source, '/');
  $absolute_source_dir = $root . '/' . $source;
  $elements = explode('/', $source);
  $project = array_pop($elements);

  // If the source is an empty directory, skip it; cvs2git barfs on these.
  if (is_empty_dir($absolute_source_dir)) {
    git_log("Skipping empty source directory '$absolute_source_dir'.", 'QUIET', $project);
    return FALSE;
  }

  if (!is_cvs_dir($absolute_source_dir)) {
    git_log("Skipping non CVS source directory '$absolute_source_dir'.", 'QUIET', $project);
    return FALSE;
  }

  // If the target destination dir exists already, remove it.
  if ($wipe && file_exists($destination) && is_dir($destination)) {
    passthru('rm -Rf ' . escapeshellarg($destination));
  }

  // Create the destination directory.
  $ret = 0;
  passthru('mkdir -p ' . escapeshellarg($destination), $ret);
  if (!empty($ret)) {
    git_log("Failed to create output directory at $destination, project import will not procede.", 'WARN', $project);
    return FALSE;
  }

  // Create a temporary directory, and register a clean up.
  $cmd = 'mktemp -dt cvs2git-import-' . escapeshellarg($project) . '.XXXXXXXXXX';
  $temp_dir = realpath(trim(`$cmd`));
  register_shutdown_function('_clean_up_import', $temp_dir);

  // Move to the temporary directory.
  chdir($temp_dir);

  // Prepare and write the option file.
  $options = array(
    '#DIR#' => $absolute_source_dir,
    '#CSV#' => dirname($config) . '/cvs.csv',
  );
  file_put_contents('./cvs2git.options', strtr(file_get_contents($config), $options));

  // Start the import process.
  git_log("Generating the fast-import dump files.", 'DEBUG', $project);
  try {
    git_invoke(escapeshellarg(CVS2GIT) . ' --options=./cvs2git.options');
  }
  catch (Exception $e) {
    git_log("cvs2git failed with error '$e'. Terminating import.", 'WARN', $project);
    return FALSE;
  }

  // Load the data into git.
  git_log("Importing project data into Git.", 'DEBUG', $project);
  git_invoke('git init', FALSE, $destination);
  try {
    git_invoke('cat tmp-cvs2git/git-blob.dat tmp-cvs2git/git-dump.dat | git fast-import --quiet', FALSE, $destination);
  }
  catch (Exception $e) {
    git_log("Fast-import failed with error '$e'. Terminating import.", 'WARN', $project);
    return FALSE;
  }

  // Do branch/tag renaming
  git_log("Performing branch/tag renaming.", 'DEBUG', $project);
  // For core
  if ($project == 'drupal' && array_search('contributions', $elements) === FALSE) { // for core
    $trans_map = $rename_patterns['core']['branches'];
    convert_project_branches($project, $destination, $trans_map);
    // Now tags.
    $trans_map = $rename_patterns['core']['tags'];
    convert_project_tags($project, $destination, $rename_patterns['core']['tagmatch'], $trans_map);
  }
  // For contrib, minus sandboxes
  else if ($elements[0] == 'contributions' && isset($elements[1]) && $elements[1] != 'sandbox') {
    // Branches first.
    $trans_map = $rename_patterns['contrib']['branches'];
    convert_project_branches($project, $destination, $trans_map);
    // Now tags.
    $trans_map = $rename_patterns['contrib']['tags'];
    convert_project_tags($project, $destination, $rename_patterns['contrib']['tagmatch'], $trans_map);
  }

  // We succeeded despite all odds!
  return TRUE;
}

/*
 * Branch/tag renaming functions ------------------------
 */

/**
 * Convert all of a contrib project's branches to the new naming convention.
 */
function convert_project_branches($project, $destination_dir, $trans_map) {
  $all_branches = $branches = array();

  try {
    $all_branches = git_invoke("ls " . escapeshellarg("$destination_dir/refs/heads/"));
    $all_branches = array_filter(explode("\n", $all_branches)); // array-ify & remove empties
  }
  catch (Exception $e) {
    git_log("Branch list retrieval failed with error '$e'.", 'WARN', $project);
  }

  if (empty($all_branches)) {
    // No branches at all, bail out.
    git_log("Project has no branches whatsoever.", 'WARN', $project);
    return;
  }

  // Kill the 'unlabeled' branches generated by cvs2git
  $unlabeleds = preg_grep('/^unlabeled/', $all_branches);
  foreach ($unlabeleds as $branch) {
    git_invoke('git branch -D ' . escapeshellarg($branch), FALSE, $destination_dir);
  }

  // Remove cvs2git junk branches from the list.
  $all_branches = array_diff($all_branches, $unlabeleds);

  // Generate a list of all valid branch names, ignoring master
  $branches = preg_grep('/^DRUPAL-/', $all_branches); // @todo be stricter?

  // Remove existing branches that have already been converted
  if (empty($branches)) {
    // No branches to work with, bail out.
    if (array_search('master', $all_branches) !== FALSE) {
      // Project has only a master branch
      git_log("Project has no conforming branches apart from master.", 'INFO', $project);
    }
    else {
      // No non-labelled branches at all. This shouldn't happen; dump the whole list if it does.
      git_log("Project has no conforming branches and no master. Full branch list: " . implode(', ', $all_branches), 'WARN', $project);
    }
    return;
  }

  // Everything needs the initial DRUPAL- stripped out.
  git_log("FULL list of the project's branches: \n" . print_r($all_branches, TRUE), 'DEBUG', $project);
  git_log("Branches in \$branches pre-transform: \n" . print_r($branches, TRUE), 'DEBUG', $project);
  $branchestmp = preg_replace(array_keys($trans_map), array_values($trans_map), $branches);
  git_log("Branches after first transform: \n" . print_r($branchestmp, TRUE), 'DEBUG', $project);
  $branches = array_diff($branches, $branchestmp);
  $new_branches = preg_replace(array_keys($trans_map), array_values($trans_map), $branches);
  git_log("Branches after second transform: \n" . print_r($new_branches, TRUE), 'DEBUG', $project);

  foreach(array_combine($branches, $new_branches) as $old_name => $new_name) {
    try {
      // Now do the rename itself. -M forces overwriting of branches.
      git_invoke("git branch -M $old_name $new_name", FALSE, $destination_dir);
    }
    catch (Exception $e) {
      // These are failing sometimes, not sure why
      git_log("Branch rename failed on branch '$old_name' with error '$e'", 'WARN', $project);
    }
  }
  verify_project_branches($project, $destination_dir, $new_branches);
}

/**
 * Verify that the project contains exactly and only the set of branches we
 * expect it to.
 */
function verify_project_branches($project, $destination_dir, $branches) {
  $all_branches = git_invoke("ls " . escapeshellarg("$destination_dir/refs/heads/"));
  $all_branches = array_filter(explode("\n", $all_branches)); // array-ify & remove empties

  if ($missing = array_diff($branches, $all_branches)) {
    git_log("Project should have the following branches after import, but does not: " . implode(', ', $missing), 'WARN', $project);
  }

  if ($nonconforming_branches = array_diff($all_branches, $branches, array('master'))) { // Ignore master
    git_log("Project has the following nonconforming branches: " . implode(', ', $nonconforming_branches), 'QUIET', $project);
  }
}

function convert_project_tags($project, $destination_dir, $match, $trans_map) {
  $all_tags = $tags = $new_tags = $nonconforming_tags = array();
  try {
    $all_tags = git_invoke('git tag -l', FALSE, $destination_dir);
    $all_tags = array_filter(explode("\n", $all_tags)); // array-ify & remove empties
  }
  catch (Exception $e) {
    git_log("Tag list retrieval failed with error '$e'", 'WARN', $project);
    return;
  }

  // Convert only tags that match naming conventions
  $tags = preg_grep($match, $all_tags);

  if (empty($tags)) {
    // No conforming tags to work with, bail out.
    if (empty($all_tags)) {
      git_log("Project has no tags at all.", 'NORMAL', $project);
    }
    else {
      git_log("Project has no conforming tags, and the following nonconforming tags: " . implode(', ', $all_tags), 'WARN', $project);
    }
    return;
  }

  // Everything needs the initial DRUPAL- stripped out.
  git_log("FULL list of the project's tags: \n" . print_r($all_tags, TRUE), 'DEBUG', $project);
  // Have to transform twice to discover tags already converted in previous runs
  git_log("Tags in \$tags pre-transform: \n" . print_r($tags, TRUE), 'DEBUG', $project);
  $tagstmp = preg_replace(array_keys($trans_map), array_values($trans_map), $tags);
  git_log("Tags after first transform: \n" . print_r($tagstmp, TRUE), 'DEBUG', $project);
  $tags = array_diff($tags, $tagstmp);
  $new_tags = preg_replace(array_keys($trans_map), array_values($trans_map), $tags);
  git_log("Tags after second transform: \n" . print_r($new_tags, TRUE), 'DEBUG', $project);

  $tag_list = array_combine($tags, $new_tags);
  foreach ($tag_list as $old_tag => $new_tag) {
    // Add the new tag.
    try {
      git_invoke("git tag -f $new_tag $old_tag", FALSE, $destination_dir);
      git_log("Created new tag '$new_tag' from old tag '$old_tag'", 'INFO', $project);
      //if ($key = array_search($new_tag, $all_tags)) {
        // existing tag - skip the rest, otherwise it'll delete the new one.
        //continue;
      //}
    }
    catch (Exception $e) {
      git_log("Creation of new tag '$new_tag' from old tag '$old_tag' failed with message $e", 'WARN', $project);
    }
    // Delete the old tag.
    try {
      git_invoke("git tag -d $old_tag", FALSE, $destination_dir);
      git_log("Deleted old tag '$old_tag'", 'INFO', $project);
    }
    catch (Exception $e) {
      git_log("Deletion of old tag '$old_tag' in project '$project' failed with message $e", 'WARN', $project);
    }
  }

  git_log("Final tag list: \n" . print_r($tag_list, TRUE), 'DEBUG', $project);

  verify_project_tags($project, $destination_dir, $tag_list);
}

/**
 * Verify that the project contains exactly and only the set of tags we
 * expect it to.
 */
function verify_project_tags($project, $destination_dir, $tags) {
  $all_tags = git_invoke('git tag -l', FALSE, $destination_dir);
  $all_tags = array_filter(explode("\n", $all_tags)); // array-ify & remove empties

  if ($missing = array_diff($tags, $all_tags)) {
    git_log("Project should have the following tags after import, but does not: " . implode(', ', $missing), 'WARN', $project);
  }

  if ($nonconforming_tags = array_diff($all_tags, $tags)) {
    git_log("Project has the following nonconforming tags: " . implode(', ', $nonconforming_tags), 'QUIET', $project);
  }
}

function cleanup_migrated_repo($project, $destination_dir, $keywords, $translations) {

  $_ENV['GIT_AUTHOR_EMAIL'] = 'tggm@no-reply.drupal.org';
  $_ENV['GIT_AUTHOR_NAME'] = 'The Great Git Migration';

  // Create a temporary directory, and register a clean up.
  $cmd = 'mktemp -dt cvs2git-import-' . escapeshellarg($project) . '.XXXXXXXXXX';
  $temp_dir = realpath(trim(`$cmd`));
  register_shutdown_function('_clean_up_import', $temp_dir);

  git_invoke("git clone $destination_dir $temp_dir");

  try {
    $all_branches = git_invoke("ls " . escapeshellarg("$destination_dir/refs/heads/"));
    $all_branches = array_filter(explode("\n", $all_branches)); // array-ify & remove empties
  }
  catch (Exception $e) {
    git_log("Branch list retrieval failed with error '$e'.", 'WARN', $project);
  }

  foreach($all_branches as $name) {
    if ($name != 'master') {
      git_invoke("git checkout -t origin/$name", FALSE, "$temp_dir/.git", $temp_dir);
    }
    else {
      git_invoke("git checkout $name", FALSE, "$temp_dir/.git", $temp_dir);
    }
    // If needed, cleanup keywords.
    if ($keywords) {
      try {
        strip_cvs_keywords($project, $temp_dir);
      }
      catch (exception $e) {
        git_log("CVS tag removal for branch $name failed with error '$e'", 'WARN', $project);
      }
    }
    // If needed, cleanup translations.
    if ($translations) {
      try {
        kill_translations($project, $temp_dir);
      }
      catch (exception $e) {
        git_log("Translation removal for branch $name failed with error '$e'", 'WARN', $project);
      }
    }
  }

  git_invoke('git push', FALSE, "$temp_dir/.git");
  return TRUE;
}

function strip_cvs_keywords($project, $directory) {
  passthru('./strip-cvs-keywords.py ' . escapeshellarg($directory));

  $commit_message = escapeshellarg("Stripping CVS keywords");
  if (git_invoke('git status --untracked-files=no -sz --', TRUE, "$directory/.git", $directory)) {
    git_invoke("git commit -a -m $commit_message", FALSE, "$directory/.git", $directory);
  }
}

function kill_translations($project, $directory) {
  $directories = git_invoke('find ' . escapeshellarg($directory) . ' -name translations -type d');
  $translations = array_filter(explode("\n", $directories)); // array-ify & remove empties
  $directories = git_invoke('find ' . escapeshellarg($directory) . ' -name po -type d');
  $po = array_filter(explode("\n", $directories)); // array-ify & remove empties

  $directories = array_merge($translations, $po);
  if (!empty($directories)) {
    $commit_message = escapeshellarg("Removing translation directories");
    foreach ($directories as $dir) {
      git_invoke("git rm -r $dir", FALSE, "$directory/.git", $directory);
    }
    git_invoke("git commit -a -m $commit_message", FALSE, "$directory/.git", $directory);
  }
}

// ------- Utility functions -----------------------------------------------

function _clean_up_import($dir) {
  git_log("Cleaning up import temp directory $dir.", 'DEBUG');
  passthru('rm -Rf ' . escapeshellarg($dir));
}
Back to Top