/contrib/tcsh/sh.hist.c

https://bitbucket.org/freebsd/freebsd-head/ · C · 1319 lines · 1039 code · 98 blank · 182 comment · 333 complexity · c24cf439788310985107d1848aeec39e MD5 · raw file

  1. /* $Header: /p/tcsh/cvsroot/tcsh/sh.hist.c,v 3.53 2011/01/24 18:10:26 christos Exp $ */
  2. /*
  3. * sh.hist.c: Shell history expansions and substitutions
  4. */
  5. /*-
  6. * Copyright (c) 1980, 1991 The Regents of the University of California.
  7. * All rights reserved.
  8. *
  9. * Redistribution and use in source and binary forms, with or without
  10. * modification, are permitted provided that the following conditions
  11. * are met:
  12. * 1. Redistributions of source code must retain the above copyright
  13. * notice, this list of conditions and the following disclaimer.
  14. * 2. Redistributions in binary form must reproduce the above copyright
  15. * notice, this list of conditions and the following disclaimer in the
  16. * documentation and/or other materials provided with the distribution.
  17. * 3. Neither the name of the University nor the names of its contributors
  18. * may be used to endorse or promote products derived from this software
  19. * without specific prior written permission.
  20. *
  21. * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND
  22. * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
  23. * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
  24. * ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE
  25. * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
  26. * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
  27. * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
  28. * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
  29. * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
  30. * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
  31. * SUCH DAMAGE.
  32. */
  33. #include "sh.h"
  34. RCSID("$tcsh: sh.hist.c,v 3.53 2011/01/24 18:10:26 christos Exp $")
  35. #include <assert.h>
  36. #include "tc.h"
  37. extern int histvalid;
  38. extern struct Strbuf histline;
  39. Char HistLit = 0;
  40. static int heq (const struct wordent *, const struct wordent *);
  41. static void hfree (struct Hist *);
  42. #define HIST_ONLY 0x01
  43. #define HIST_SAVE 0x02
  44. #define HIST_LOAD 0x04
  45. #define HIST_REV 0x08
  46. #define HIST_CLEAR 0x10
  47. #define HIST_MERGE 0x20
  48. #define HIST_TIME 0x40
  49. /*
  50. * C shell
  51. */
  52. /* Static functions don't show up in gprof summaries. So eliminate "static"
  53. * modifier from some frequently called functions. */
  54. #ifdef PROF
  55. #define PG_STATIC
  56. #else
  57. #define PG_STATIC static
  58. #endif
  59. /* #define DEBUG_HIST 1 */
  60. static const int fastMergeErase = 1;
  61. static unsigned histCount = 0; /* number elements on history list */
  62. static struct Hist *histTail = NULL; /* last element on history list */
  63. static struct Hist *histMerg = NULL; /* last element merged by Htime */
  64. static void insertHistHashTable(struct Hist *, unsigned);
  65. /* Insert new element (hp) in history list after specified predecessor (pp). */
  66. static void
  67. hinsert(struct Hist *hp, struct Hist *pp)
  68. {
  69. struct Hist *fp = pp->Hnext; /* following element, if any */
  70. hp->Hnext = fp, hp->Hprev = pp;
  71. pp->Hnext = hp;
  72. if (fp)
  73. fp->Hprev = hp;
  74. else
  75. histTail = hp; /* meaning hp->Hnext == NULL */
  76. histCount++;
  77. }
  78. /* Remove the entry from the history list. */
  79. static void
  80. hremove(struct Hist *hp)
  81. {
  82. struct Hist *pp = hp->Hprev;
  83. assert(pp); /* elements always have a previous */
  84. pp->Hnext = hp->Hnext;
  85. if (hp->Hnext)
  86. hp->Hnext->Hprev = pp;
  87. else
  88. histTail = pp; /* we must have been last */
  89. if (hp == histMerg) /* deleting this hint from list */
  90. histMerg = NULL;
  91. assert(histCount > 0);
  92. histCount--;
  93. }
  94. /* Prune length of history list to specified size by history variable. */
  95. PG_STATIC void
  96. discardExcess(int histlen)
  97. {
  98. struct Hist *hp, *np;
  99. if (histTail == NULL) {
  100. assert(histCount == 0);
  101. return; /* no entries on history list */
  102. }
  103. /* Prune dummy entries from the front, then old entries from the back. If
  104. * the list is still too long scan the whole list as before. But only do a
  105. * full scan if the list is more than 6% (1/16th) too long. */
  106. while (histCount > (unsigned)histlen && (np = Histlist.Hnext)) {
  107. if (eventno - np->Href >= histlen || histlen == 0)
  108. hremove(np), hfree(np);
  109. else
  110. break;
  111. }
  112. while (histCount > (unsigned)histlen && (np = histTail) != &Histlist) {
  113. if (eventno - np->Href >= histlen || histlen == 0)
  114. hremove(np), hfree(np);
  115. else
  116. break;
  117. }
  118. if (histCount - (histlen >> 4) <= (unsigned)histlen)
  119. return; /* don't bother doing the full scan */
  120. for (hp = &Histlist; histCount > (unsigned)histlen &&
  121. (np = hp->Hnext) != NULL;)
  122. if (eventno - np->Href >= histlen || histlen == 0)
  123. hremove(np), hfree(np);
  124. else
  125. hp = np;
  126. }
  127. /* Add the command "sp" to the history list. */
  128. void
  129. savehist(
  130. struct wordent *sp,
  131. int mflg) /* true if -m (merge) specified */
  132. {
  133. int histlen = 0;
  134. Char *cp;
  135. /* throw away null lines */
  136. if (sp && sp->next->word[0] == '\n')
  137. return;
  138. cp = varval(STRhistory);
  139. while (*cp) {
  140. if (!Isdigit(*cp)) {
  141. histlen = 0;
  142. break;
  143. }
  144. histlen = histlen * 10 + *cp++ - '0';
  145. }
  146. if (sp)
  147. (void) enthist(++eventno, sp, 1, mflg, histlen);
  148. discardExcess(histlen);
  149. }
  150. #define USE_JENKINS_HASH 1
  151. /* #define USE_ONE_AT_A_TIME 1 */
  152. #undef PRIME_LENGTH /* no need for good HTL */
  153. #ifdef USE_JENKINS_HASH
  154. #define hashFcnName "lookup3"
  155. /* From:
  156. lookup3.c, by Bob Jenkins, May 2006, Public Domain.
  157. "... You can use this free for any purpose. It's in
  158. the public domain. It has no warranty."
  159. http://burtleburtle.net/bob/hash/index.html
  160. */
  161. #define rot(x,k) (((x)<<(k)) | ((x)>>(32-(k))))
  162. #define mix(a,b,c) \
  163. { \
  164. a -= c; a ^= rot(c, 4); c += b; \
  165. b -= a; b ^= rot(a, 6); a += c; \
  166. c -= b; c ^= rot(b, 8); b += a; \
  167. a -= c; a ^= rot(c,16); c += b; \
  168. b -= a; b ^= rot(a,19); a += c; \
  169. c -= b; c ^= rot(b, 4); b += a; \
  170. }
  171. #define final(a,b,c) \
  172. { \
  173. c ^= b; c -= rot(b,14); \
  174. a ^= c; a -= rot(c,11); \
  175. b ^= a; b -= rot(a,25); \
  176. c ^= b; c -= rot(b,16); \
  177. a ^= c; a -= rot(c, 4); \
  178. b ^= a; b -= rot(a,14); \
  179. c ^= b; c -= rot(b,24); \
  180. }
  181. struct hashValue /* State used to hash a wordend word list. */
  182. {
  183. uint32_t a, b, c;
  184. };
  185. /* Set up the internal state */
  186. static void
  187. initializeHash(struct hashValue *h)
  188. {
  189. h->a = h->b = h->c = 0xdeadbeef;
  190. }
  191. /* This does a partial hash of the Chars in a single word. For efficiency we
  192. * include 3 versions of the code to pack Chars into 32-bit words for the
  193. * mixing function. */
  194. static void
  195. addWordToHash(struct hashValue *h, const Char *word)
  196. {
  197. uint32_t a = h->a, b = h->b, c = h->c;
  198. #ifdef SHORT_STRINGS
  199. #ifdef WIDE_STRINGS
  200. assert(sizeof(Char) >= 4);
  201. while (1) {
  202. unsigned k;
  203. if ((k = (uChar)*word++) == 0) break; a += k;
  204. if ((k = (uChar)*word++) == 0) break; b += k;
  205. if ((k = (uChar)*word++) == 0) break; c += k;
  206. mix(a, b, c);
  207. }
  208. #else
  209. assert(sizeof(Char) == 2);
  210. while (1) {
  211. unsigned k;
  212. if ((k = (uChar)*word++) == 0) break; a += k;
  213. if ((k = (uChar)*word++) == 0) break; a += k << 16;
  214. if ((k = (uChar)*word++) == 0) break; b += k;
  215. if ((k = (uChar)*word++) == 0) break; b += k << 16;
  216. if ((k = (uChar)*word++) == 0) break; c += k;
  217. if ((k = (uChar)*word++) == 0) break; c += k << 16;
  218. mix(a, b, c);
  219. }
  220. #endif
  221. #else
  222. assert(sizeof(Char) == 1);
  223. while (1) {
  224. unsigned k;
  225. if ((k = *word++) == 0) break; a += k;
  226. if ((k = *word++) == 0) break; a += k << 8;
  227. if ((k = *word++) == 0) break; a += k << 16;
  228. if ((k = *word++) == 0) break; a += k << 24;
  229. if ((k = *word++) == 0) break; b += k;
  230. if ((k = *word++) == 0) break; b += k << 8;
  231. if ((k = *word++) == 0) break; b += k << 16;
  232. if ((k = *word++) == 0) break; b += k << 24;
  233. if ((k = *word++) == 0) break; c += k;
  234. if ((k = *word++) == 0) break; c += k << 8;
  235. if ((k = *word++) == 0) break; c += k << 16;
  236. if ((k = *word++) == 0) break; c += k << 24;
  237. mix(a, b, c);
  238. }
  239. #endif
  240. h->a = a, h->b = b, h->c = c;
  241. }
  242. static void
  243. addCharToHash(struct hashValue *h, Char ch)
  244. {
  245. /* The compiler (gcc -O2) seems to do a good job optimizing this without
  246. * explicitly extracting into local variables. */
  247. h->a += (uChar)ch;
  248. mix(h->a, h->b, h->c);
  249. }
  250. static uint32_t
  251. finalizeHash(struct hashValue *h)
  252. {
  253. uint32_t a = h->a, b = h->b, c = h->c;
  254. final(a, b, c);
  255. return c;
  256. }
  257. #elif USE_ONE_AT_A_TIME
  258. #define hashFcnName "one-at-a-time"
  259. /* This one is also from Bob Jenkins, but is slower but simpler than lookup3.
  260. "... The code given here are all public domain."
  261. http://burtleburtle.net/bob/hash/doobs.html */
  262. #if 0
  263. ub4
  264. one_at_a_time(char *key, ub4 len)
  265. {
  266. ub4 hash, i;
  267. for (hash=0, i=0; i<len; ++i)
  268. {
  269. hash += key[i];
  270. hash += (hash << 10);
  271. hash ^= (hash >> 6);
  272. }
  273. hash += (hash << 3);
  274. hash ^= (hash >> 11);
  275. hash += (hash << 15);
  276. return (hash & mask);
  277. }
  278. #endif
  279. struct hashValue { uint32_t h; };
  280. static void
  281. initializeHash(struct hashValue *h)
  282. {
  283. h->h = 0;
  284. }
  285. static void
  286. addWordToHash(struct hashValue *h, const Char *word)
  287. {
  288. unsigned k;
  289. uint32_t hash = h->h;
  290. while (k = (uChar)*word++)
  291. hash += k, hash += hash << 10, hash ^= hash >> 6;
  292. h->h = hash;
  293. }
  294. static void
  295. addCharToHash(struct hashValue *h, Char c)
  296. {
  297. Char b[2] = { c, 0 };
  298. addWordToHash(h, b);
  299. }
  300. static uint32_t
  301. finalizeHash(struct hashValue *h)
  302. {
  303. unsigned hash = h->h;
  304. hash += (hash << 3);
  305. hash ^= (hash >> 11);
  306. hash += (hash << 15);
  307. return hash;
  308. }
  309. #else
  310. #define hashFcnName "add-mul"
  311. /* Simple multipy and add hash. */
  312. #define PRIME_LENGTH 1 /* need "good" HTL */
  313. struct hashValue { uint32_t h; };
  314. static void
  315. initializeHash(struct hashValue *h)
  316. {
  317. h->h = 0xe13e2345;
  318. }
  319. static void
  320. addWordToHash(struct hashValue *h, const Char *word)
  321. {
  322. unsigned k;
  323. uint32_t hash = h->h;
  324. while (k = (uChar)*word++)
  325. hash = hash * 0x9e4167b9 + k;
  326. h->h = hash;
  327. }
  328. static void
  329. addCharToHash(struct hashValue *h, Char c)
  330. {
  331. h->h = h->h * 0x9e4167b9 + (uChar)c;
  332. }
  333. static uint32_t
  334. finalizeHash(struct hashValue *h)
  335. {
  336. return h->h;
  337. }
  338. #endif
  339. static unsigned
  340. hashhist(struct wordent *h0)
  341. {
  342. struct hashValue s;
  343. struct wordent *firstWord = h0->next;
  344. struct wordent *h = firstWord;
  345. unsigned hash = 0;
  346. initializeHash(&s);
  347. for (; h != h0; h = h->next) {
  348. if (h->word[0] == '\n')
  349. break; /* don't hash newline */
  350. if (h != firstWord)
  351. addCharToHash(&s, ' '); /* space between words */
  352. addWordToHash(&s, h->word);
  353. }
  354. hash = finalizeHash(&s);
  355. /* Zero means no hash value, so never return zero as a hash value. */
  356. return hash ? hash : 0x7fffffff; /* prime! */
  357. }
  358. #if 0
  359. unsigned
  360. hashStr(Char *str)
  361. {
  362. struct hashValue s;
  363. initializeHash(&s);
  364. addWordToHash(&s, str);
  365. return finalizeHash(&s);
  366. }
  367. #endif
  368. #ifdef PRIME_LENGTH /* need good HTL */
  369. #define hash2tableIndex(hash, len) ((hash) % len)
  370. #else
  371. #define hash2tableIndex(hash, len) ((hash) & (len-1))
  372. #endif
  373. /* This code can be enabled to test the above hash functions for speed and
  374. * collision avoidance. The testing is enabled by "occasional" calls to
  375. * displayHistStats(), see which. */
  376. #ifdef DEBUG_HIST
  377. #ifdef BSDTIMES
  378. static double
  379. doTiming(int start) {
  380. static struct timeval beginTime;
  381. if (start) {
  382. gettimeofday(&beginTime, NULL);
  383. return 0.0;
  384. } else {
  385. struct timeval now;
  386. gettimeofday(&now, NULL);
  387. return (now.tv_sec-beginTime.tv_sec) +
  388. (now.tv_usec-beginTime.tv_usec)/1e6;
  389. }
  390. }
  391. #else
  392. static double
  393. doTiming(int start) {
  394. USE(start);
  395. return 0.0;
  396. }
  397. #endif
  398. static void
  399. generateHashes(int nChars, unsigned nWords, unsigned samples, unsigned *hashes,
  400. unsigned length)
  401. {
  402. if (nChars < 1)
  403. return;
  404. nWords = (nWords < 1) ? 1 : (nWords > 4) ? 4 : nWords;
  405. Char *number = xmalloc((nChars+nWords)*sizeof(Char));
  406. struct wordent word[4];
  407. struct wordent base = { NULL, &word[0], &word[0] };
  408. word[0].word = number, word[0].next = &base, word[0].prev = &base;
  409. unsigned w = 0; /* word number */
  410. /* Generate multiple words of length 2, 3, 5, then all the rest. */
  411. unsigned wBoundaries[4] = { 2-1, 2+3-1, 2+3+5-1, 0 };
  412. /* Ensure the last word has at least 4 Chars in it. */
  413. while (nWords >= 2 && nChars < (wBoundaries[nWords-2]+1) + 4)
  414. nWords--;
  415. wBoundaries[nWords-1] = 0xffffffff; /* don't end word past this point */
  416. unsigned i;
  417. for (i = 0; i<nChars; i++) {
  418. /* In deference to the gawd awful add-mul hash, we won't use the worse
  419. * case here (setting all Chars to 1), but assume mostly (or at least
  420. * initially) ASCII data. */
  421. number[i+w] = '!'; /* 0x21 = 33 */
  422. if (i == wBoundaries[w]) { /* end a word here and move to next */
  423. w++; /* next word */
  424. number[i+w] = 0; /* terminate */
  425. word[w].word = &number[i+w+1];
  426. word[w].next = &base, word[w].prev = &word[w-1];
  427. word[w-1].next = &word[w], base.prev = &word[w];
  428. }
  429. }
  430. /* w is the index of the last word actually created. */
  431. number[nChars + w] = 0; /* terminate last word */
  432. unsigned timeLimit = !samples;
  433. if (samples == 0)
  434. samples = 1000000000;
  435. doTiming(1);
  436. double sec;
  437. for (i = 0; i < samples; i++) {
  438. /* increment 4 digit base 255 number; last characters vary fastest */
  439. unsigned j = nChars-1 + w;
  440. while (1) {
  441. if (++number[j] != 0)
  442. break;
  443. /* else reset this digit and proceed to next one */
  444. number[j] = 1;
  445. if (&number[j] <= word[w].word)
  446. break; /* stop at beginning of last word */
  447. j--;
  448. }
  449. if (word[w].word[0] == '\n')
  450. word[w].word[0]++; /* suppress newline character */
  451. unsigned hash = hashhist(&base);
  452. hashes[hash2tableIndex(hash, length)]++;
  453. if (timeLimit && (i & 0x3ffff) == 0x3ffff) {
  454. sec = doTiming(0);
  455. if (sec > 10)
  456. break;
  457. }
  458. }
  459. if (i >= samples)
  460. sec = doTiming(0);
  461. else
  462. samples = i; /* number we actually did */
  463. if (sec > 0.01) {
  464. xprintf("Hash %d (%d Char %u words) with %s: %d nsec/hash, %d mcps\n",
  465. samples, nChars, w+1, hashFcnName, (int)((sec/samples)*1e9),
  466. (int)((double)samples*nChars/sec/1e6));
  467. }
  468. }
  469. #endif /* DEBUG_HIST */
  470. #ifdef DEBUG_HIST
  471. static void
  472. testHash(void)
  473. {
  474. static const Char STRtestHashTimings[] =
  475. { 't','e','s','t','H','a','s','h','T','i','m','i','n','g','s', 0 };
  476. struct varent *vp = adrof(STRtestHashTimings);
  477. if (vp && vp->vec) {
  478. unsigned hashes[4]; /* dummy place to put hashes */
  479. Char **vals = vp->vec;
  480. while (*vals) {
  481. int length = getn(*vals);
  482. unsigned words =
  483. (length < 5) ? 1 : (length < 25) ? 2 : (length < 75) ? 3 : 4;
  484. if (length > 0)
  485. generateHashes(length, words, 0, hashes, 4);
  486. vals++;
  487. }
  488. }
  489. unsigned length = 1024;
  490. #ifdef PRIME_LENGTH /* need good HTL */
  491. length = 1021;
  492. #endif
  493. unsigned *hashes = xmalloc(length*sizeof(unsigned));
  494. memset(hashes, 0, length*sizeof(unsigned));
  495. /* Compute collision statistics for half full hashes modulo "length". */
  496. generateHashes(4, 1, length/2, hashes, length);
  497. /* Evaluate collisions by comparing occupancy rates (mean value 0.5).
  498. * One bin for each number of hits. */
  499. unsigned bins[155];
  500. memset(bins, 0, sizeof(bins));
  501. unsigned highest = 0;
  502. unsigned i;
  503. for (i = 0; i<length; i++) {
  504. unsigned hits = hashes[i];
  505. if (hits >= sizeof(bins)/sizeof(bins[0])) /* clip */
  506. hits = highest = sizeof(bins)/sizeof(bins[0]) - 1;
  507. if (hits > highest)
  508. highest = hits;
  509. bins[hits]++;
  510. }
  511. xprintf("Occupancy of %d buckets by %d hashes %d Chars %d word with %s\n",
  512. length, length/2, 4, 1, hashFcnName);
  513. for (i = 0; i <= highest; i++) {
  514. xprintf(" %d buckets (%d%%) with %d hits\n",
  515. bins[i], bins[i]*100/length, i);
  516. }
  517. /* Count run lengths to evaluate linear rehashing effectiveness. Estimate
  518. * a little corrupted by edge effects. */
  519. memset(bins, 0, sizeof(bins));
  520. highest = 0;
  521. for (i = 0; hashes[i] == 0; i++); /* find first occupied bucket */
  522. unsigned run = 0;
  523. unsigned rehashed = 0;
  524. for (; i<length; i++) {
  525. unsigned hits = hashes[i];
  526. if (hits == 0 && rehashed > 0)
  527. hits = 1 && rehashed--;
  528. else if (hits > 1)
  529. rehashed += hits-1;
  530. if (hits)
  531. run++;
  532. else {
  533. /* a real free slot, count it */
  534. if (run >= sizeof(bins)/sizeof(bins[0])) /* clip */
  535. run = highest = sizeof(bins)/sizeof(bins[0]) - 1;
  536. if (run > highest)
  537. highest = run;
  538. bins[run]++;
  539. run = 0;
  540. }
  541. }
  542. /* Ignore the partial run at end as we ignored the beginning. */
  543. double merit = 0.0, entries = 0;
  544. for (i = 0; i <= highest; i++) {
  545. entries += bins[i]*i; /* total hashed objects */
  546. merit += bins[i]*i*i;
  547. }
  548. xprintf("Rehash collision figure of merit %u (ideal=100), run lengths:\n",
  549. (int)(100.0*merit/entries));
  550. for (i = 0; i <= highest; i++) {
  551. if (bins[i] != 0)
  552. xprintf(" %d runs of length %d buckets\n", bins[i], i);
  553. }
  554. xfree(hashes);
  555. }
  556. #endif /* DEBUG_HIST */
  557. /* Compares two word lists for equality. */
  558. static int
  559. heq(const struct wordent *a0, const struct wordent *b0)
  560. {
  561. const struct wordent *a = a0->next, *b = b0->next;
  562. for (;;) {
  563. if (Strcmp(a->word, b->word) != 0)
  564. return 0;
  565. a = a->next;
  566. b = b->next;
  567. if (a == a0)
  568. return (b == b0) ? 1 : 0;
  569. if (b == b0)
  570. return 0;
  571. }
  572. }
  573. /* Renumber entries following p, which we will be deleting. */
  574. PG_STATIC void
  575. renumberHist(struct Hist *p)
  576. {
  577. int n = p->Href;
  578. while ((p = p->Hnext))
  579. p->Href = n--;
  580. }
  581. /* The hash table is implemented as an array of pointers to Hist entries. Each
  582. * entry is located in the table using hash2tableIndex() and checking the
  583. * following entries in case of a collision (linear rehash). Free entries in
  584. * the table are zero (0, NULL, emptyHTE). Deleted entries that cannot yet be
  585. * freed are set to one (deletedHTE). The Hist.Hhash member is non-zero iff
  586. * the entry is in the hash table. When the hash table get too full, it is
  587. * reallocated to be approximately twice the history length (see
  588. * getHashTableSize). */
  589. static struct Hist **histHashTable = NULL;
  590. static unsigned histHashTableLength = 0; /* number of Hist pointers in table */
  591. static struct Hist * const emptyHTE = NULL;
  592. static struct Hist * const deletedHTE = (struct Hist *)1;
  593. static struct {
  594. unsigned insertCount;
  595. unsigned removeCount;
  596. unsigned rehashes;
  597. int deleted;
  598. } hashStats;
  599. #ifdef DEBUG_HIST
  600. void
  601. checkHistHashTable(int print)
  602. {
  603. unsigned occupied = 0;
  604. unsigned deleted = 0;
  605. unsigned i;
  606. for (i = 0; i<histHashTableLength; i++)
  607. if (histHashTable[i] == emptyHTE)
  608. continue;
  609. else if (histHashTable[i] == deletedHTE)
  610. deleted++;
  611. else
  612. occupied++;
  613. if (print)
  614. xprintf(" found len %u occupied %u deleted %u\n",
  615. histHashTableLength, occupied, deleted);
  616. assert(deleted == hashStats.deleted);
  617. }
  618. static int doneTest = 0;
  619. /* Main entry point for displaying history statistics and hash function
  620. * behavior. */
  621. void
  622. displayHistStats(const char *reason)
  623. {
  624. /* Just hash statistics for now. */
  625. xprintf("%s history hash table len %u count %u (deleted %d)\n", reason,
  626. histHashTableLength, histCount, hashStats.deleted);
  627. xprintf(" inserts %u rehashes %u%% each\n",
  628. hashStats.insertCount,
  629. (hashStats.insertCount
  630. ? 100*hashStats.rehashes/hashStats.insertCount : 0));
  631. xprintf(" removes %u net %u\n",
  632. hashStats.removeCount,
  633. hashStats.insertCount - hashStats.removeCount);
  634. assert(hashStats.insertCount >= hashStats.removeCount);
  635. checkHistHashTable(1);
  636. memset(&hashStats, 0, sizeof(hashStats));
  637. if (!doneTest) {
  638. testHash();
  639. doneTest = 1;
  640. }
  641. }
  642. #else
  643. void
  644. displayHistStats(const char *reason)
  645. {
  646. USE(reason);
  647. }
  648. #endif
  649. static void
  650. discardHistHashTable(void)
  651. {
  652. if (histHashTable == NULL)
  653. return;
  654. displayHistStats("Discarding");
  655. xfree(histHashTable);
  656. histHashTable = NULL;
  657. }
  658. /* Computes a new hash table size, when the current one is too small. */
  659. static unsigned
  660. getHashTableSize(int histlen)
  661. {
  662. unsigned target = histlen * 2;
  663. unsigned e = 5;
  664. unsigned size;
  665. while ((size = 1<<e) < target)
  666. e++;
  667. #ifdef PRIME_LENGTH /* need good HTL */
  668. /* Not all prime, but most are and none have factors smaller than 11. */
  669. return size+15;
  670. #else
  671. assert((size & (size-1)) == 0); /* must be a power of two */
  672. return size;
  673. #endif
  674. }
  675. /* Create the hash table or resize, if necessary. */
  676. static void
  677. createHistHashTable(int histlen)
  678. {
  679. if (histlen == 0) {
  680. discardHistHashTable();
  681. return;
  682. }
  683. if (histlen < 0) {
  684. histlen = getn(varval(STRhistory));
  685. if (histlen == 0)
  686. return; /* no need for hash table */
  687. assert(histlen > 0);
  688. }
  689. if (histHashTable != NULL) {
  690. if (histCount < histHashTableLength * 3 / 4)
  691. return; /* good enough for now */
  692. discardHistHashTable(); /* too small */
  693. }
  694. histHashTableLength = getHashTableSize(
  695. histlen > (int)histCount ? histlen : (int)histCount);
  696. histHashTable = xmalloc(histHashTableLength * sizeof(struct Hist *));
  697. memset(histHashTable, 0, histHashTableLength * sizeof(struct Hist *));
  698. assert(histHashTable[0] == emptyHTE);
  699. /* Now insert all the entries on the history list into the hash table. */
  700. {
  701. struct Hist *hp;
  702. for (hp = &Histlist; (hp = hp->Hnext) != NULL;) {
  703. unsigned lpHash = hashhist(&hp->Hlex);
  704. assert(!hp->Hhash || hp->Hhash == lpHash);
  705. hp->Hhash = 0; /* force insert to new hash table */
  706. insertHistHashTable(hp, lpHash);
  707. }
  708. }
  709. }
  710. /* Insert np into the hash table. We assume that np is already on the
  711. * Histlist. The specified hashval matches the new Hist entry but has not yet
  712. * been assigned to Hhash (or the element is already on the hash table). */
  713. static void
  714. insertHistHashTable(struct Hist *np, unsigned hashval)
  715. {
  716. unsigned rehashes = 0;
  717. unsigned hi = 0;
  718. if (!histHashTable)
  719. return;
  720. if (np->Hhash != 0) {
  721. /* already in hash table */
  722. assert(hashval == np->Hhash);
  723. return;
  724. }
  725. assert(np != deletedHTE);
  726. /* Find a free (empty or deleted) slot, using linear rehash. */
  727. assert(histHashTable);
  728. for (rehashes = 0;
  729. ((hi = hash2tableIndex(hashval + rehashes, histHashTableLength)),
  730. histHashTable[hi] != emptyHTE && histHashTable[hi] != deletedHTE);
  731. rehashes++) {
  732. assert(np != histHashTable[hi]);
  733. if (rehashes >= histHashTableLength / 10) {
  734. /* Hash table is full, so grow it. We assume the create function
  735. * will roughly double the size we give it. Create initializes the
  736. * new table with everything on the Histlist, so we are done when
  737. * it returns. */
  738. #ifdef DEBUG_HIST
  739. xprintf("Growing history hash table from %d ...",
  740. histHashTableLength);
  741. flush();
  742. #endif
  743. discardHistHashTable();
  744. createHistHashTable(histHashTableLength);
  745. #ifdef DEBUG_HIST
  746. xprintf("to %d.\n", histHashTableLength);
  747. #endif
  748. return;
  749. }
  750. }
  751. /* Might be sensible to grow hash table if rehashes is "too big" here. */
  752. if (histHashTable[hi] == deletedHTE)
  753. hashStats.deleted--;
  754. histHashTable[hi] = np;
  755. np->Hhash = hashval;
  756. hashStats.insertCount++;
  757. hashStats.rehashes += rehashes;
  758. }
  759. /* Remove the 'np' entry from the hash table. */
  760. static void
  761. removeHistHashTable(struct Hist *np)
  762. {
  763. unsigned hi = np->Hhash;
  764. if (!histHashTable || !hi)
  765. return; /* no hash table or not on it */
  766. /* find desired entry */
  767. while ((hi = hash2tableIndex(hi, histHashTableLength)),
  768. histHashTable[hi] != emptyHTE) {
  769. if (np == histHashTable[hi]) {
  770. unsigned i;
  771. unsigned deletes = 0;
  772. histHashTable[hi] = deletedHTE; /* dummy, but non-zero entry */
  773. /* now peek ahead to see if the dummies are really necessary. */
  774. i = 1;
  775. while (histHashTable[hash2tableIndex(hi+i, histHashTableLength)] ==
  776. deletedHTE)
  777. i++;
  778. if (histHashTable[hash2tableIndex(hi+i, histHashTableLength)] ==
  779. emptyHTE) {
  780. /* dummies are no longer necessary placeholders. */
  781. deletes = i;
  782. while (i-- > 0) {
  783. histHashTable[hash2tableIndex(hi+i, histHashTableLength)] =
  784. emptyHTE;
  785. }
  786. }
  787. hashStats.deleted += 1 - deletes; /* delta deleted entries */
  788. hashStats.removeCount++;
  789. return;
  790. }
  791. hi++; /* linear rehash */
  792. }
  793. assert(!"Hist entry not found in hash table");
  794. }
  795. /* Search the history hash table for a command matching lp, using hashval as
  796. * its hash value. */
  797. static struct Hist *
  798. findHistHashTable(struct wordent *lp, unsigned hashval)
  799. {
  800. unsigned deleted = 0; /* number of deleted entries skipped */
  801. unsigned hi = hashval;
  802. struct Hist *hp;
  803. if (!histHashTable)
  804. return NULL;
  805. while ((hi = hash2tableIndex(hi, histHashTableLength)),
  806. (hp = histHashTable[hi]) != emptyHTE) {
  807. if (hp == deletedHTE)
  808. deleted++;
  809. else if (hp->Hhash == hashval && heq(lp, &(hp->Hlex)))
  810. return hp;
  811. if (deleted > (histHashTableLength>>4)) {
  812. /* lots of deletes, so we need a sparser table. */
  813. discardHistHashTable();
  814. createHistHashTable(histHashTableLength);
  815. return findHistHashTable(lp, hashval);
  816. }
  817. hi++; /* linear rehash */
  818. }
  819. return NULL;
  820. }
  821. /* When merge semantics are in use, find the approximate predecessor for the
  822. * new entry, so that the Htime entries are decreasing. Return the entry just
  823. * before the first entry with equal times, so the caller can check for
  824. * duplicates. When pTime is not NULL, use it as a starting point for search,
  825. * otherwise search from beginning (largest time value) of history list. */
  826. PG_STATIC struct Hist *
  827. mergeInsertionPoint(
  828. struct Hist *np, /* new entry to be inserted */
  829. struct Hist *pTime) /* hint about where to insert */
  830. {
  831. struct Hist *pp, *p;
  832. if (histTail && histTail->Htime >= np->Htime)
  833. pTime = histTail; /* new entry goes at the end */
  834. if (histMerg && histMerg != &Histlist && histMerg != Histlist.Hnext) {
  835. /* Check above and below previous insertion point, in case we're adding
  836. * sequential times in the middle of the list (e.g. history -M). */
  837. if (histMerg->Htime >= np->Htime)
  838. pTime = histMerg;
  839. else if (histMerg->Hprev->Htime >= np->Htime)
  840. pTime = histMerg->Hprev;
  841. }
  842. if (pTime) {
  843. /* With hint, search up the list until Htime is greater. We skip past
  844. * the equal ones, too, so our caller can elide duplicates. */
  845. pp = pTime;
  846. while (pp != &Histlist && pp->Htime <= np->Htime)
  847. pp = pp->Hprev;
  848. } else
  849. pp = &Histlist;
  850. /* Search down the list while current entry's time is too large. */
  851. while ((p = pp->Hnext) && (p->Htime > np->Htime))
  852. pp = p; /* advance insertion point */
  853. /* Remember recent position as hint for next time */
  854. histMerg = pp;
  855. return pp;
  856. }
  857. /* Bubble Hnum & Href in new entry down to pp through earlier part of list. */
  858. PG_STATIC void bubbleHnumHrefDown(struct Hist *np, struct Hist *pp)
  859. {
  860. struct Hist *p;
  861. for (p = Histlist.Hnext; p != pp->Hnext; p = p->Hnext) {
  862. /* swap Hnum & Href values of p and np. */
  863. int n = p->Hnum, r = p->Href;
  864. p->Hnum = np->Hnum; p->Href = np->Href;
  865. np->Hnum = n; np->Href = r;
  866. }
  867. }
  868. /* Enter new command into the history list according to current settings. */
  869. struct Hist *
  870. enthist(
  871. int event, /* newly incremented global eventno */
  872. struct wordent *lp,
  873. int docopy,
  874. int mflg, /* true if merge requested */
  875. int histlen) /* -1 if unknown */
  876. {
  877. struct Hist *p = NULL, *pp = &Histlist, *pTime = NULL;
  878. struct Hist *np;
  879. const Char *dp;
  880. unsigned lpHash = 0; /* non-zero if hashing entries */
  881. if ((dp = varval(STRhistdup)) != STRNULL) {
  882. if (eq(dp, STRerase)) {
  883. /* masaoki@akebono.tky.hp.com (Kobayashi Masaoki) */
  884. createHistHashTable(histlen);
  885. lpHash = hashhist(lp);
  886. assert(lpHash != 0);
  887. p = findHistHashTable(lp, lpHash);
  888. if (p) {
  889. if (Htime != 0 && p->Htime > Htime)
  890. Htime = p->Htime;
  891. /* If we are merging, and the old entry is at the place we want
  892. * to insert the new entry, then remember the place. */
  893. if (mflg && Htime != 0 && p->Hprev->Htime >= Htime)
  894. pTime = p->Hprev;
  895. if (!fastMergeErase)
  896. renumberHist(p); /* Reset Href of subsequent entries */
  897. hremove(p);
  898. hfree(p);
  899. p = NULL; /* so new entry is allocated below */
  900. }
  901. }
  902. else if (eq(dp, STRall)) {
  903. createHistHashTable(histlen);
  904. lpHash = hashhist(lp);
  905. assert(lpHash != 0);
  906. p = findHistHashTable(lp, lpHash);
  907. if (p) /* p!=NULL, only update this entry's Htime below */
  908. eventno--; /* not adding a new event */
  909. }
  910. else if (eq(dp, STRprev)) {
  911. if (pp->Hnext && heq(lp, &(pp->Hnext->Hlex))) {
  912. p = pp->Hnext;
  913. eventno--;
  914. }
  915. }
  916. }
  917. np = p ? p : xmalloc(sizeof(*np));
  918. /* Pick up timestamp set by lex() in Htime if reading saved history */
  919. if (Htime != 0) {
  920. np->Htime = Htime;
  921. Htime = 0;
  922. }
  923. else
  924. (void) time(&(np->Htime));
  925. if (p == np)
  926. return np; /* reused existing entry */
  927. /* Initialize the new entry. */
  928. np->Hnum = np->Href = event;
  929. if (docopy) {
  930. copylex(&np->Hlex, lp);
  931. if (histvalid)
  932. np->histline = Strsave(histline.s);
  933. else
  934. np->histline = NULL;
  935. }
  936. else {
  937. np->Hlex.next = lp->next;
  938. lp->next->prev = &np->Hlex;
  939. np->Hlex.prev = lp->prev;
  940. lp->prev->next = &np->Hlex;
  941. np->histline = NULL;
  942. }
  943. np->Hhash = 0;
  944. /* The head of history list is the default insertion point.
  945. If merging, advance insertion point, in pp, according to Htime. */
  946. /* XXX -- In histdup=all, Htime values can be non-monotonic. */
  947. if (mflg) { /* merge according to np->Htime */
  948. pp = mergeInsertionPoint(np, pTime);
  949. for (p = pp->Hnext; p && p->Htime == np->Htime; pp = p, p = p->Hnext) {
  950. if (heq(&p->Hlex, &np->Hlex)) {
  951. eventno--; /* duplicate, so don't add new event */
  952. hfree(np);
  953. return (p);
  954. }
  955. }
  956. /* pp is now the last entry with time >= to np. */
  957. if (!fastMergeErase) { /* renumber at end of loadhist */
  958. /* Before inserting np after pp, bubble its Hnum & Href values down
  959. * through the earlier part of list. */
  960. bubbleHnumHrefDown(np, pp);
  961. }
  962. }
  963. else
  964. pp = &Histlist; /* insert at beginning of history */
  965. hinsert(np, pp);
  966. if (lpHash && histlen != 0) /* erase & all modes use hash table */
  967. insertHistHashTable(np, lpHash);
  968. else
  969. discardHistHashTable();
  970. return (np);
  971. }
  972. static void
  973. hfree(struct Hist *hp)
  974. {
  975. assert(hp != histMerg);
  976. if (hp->Hhash)
  977. removeHistHashTable(hp);
  978. freelex(&hp->Hlex);
  979. if (hp->histline)
  980. xfree(hp->histline);
  981. xfree(hp);
  982. }
  983. PG_STATIC void
  984. phist(struct Hist *hp, int hflg)
  985. {
  986. if (hflg & HIST_ONLY) {
  987. int old_output_raw;
  988. /*
  989. * Control characters have to be written as is (output_raw).
  990. * This way one can preserve special characters (like tab) in
  991. * the history file.
  992. * From: mveksler@vnet.ibm.com (Veksler Michael)
  993. */
  994. old_output_raw = output_raw;
  995. output_raw = 1;
  996. cleanup_push(&old_output_raw, output_raw_restore);
  997. if (hflg & HIST_TIME)
  998. /*
  999. * Make file entry with history time in format:
  1000. * "+NNNNNNNNNN" (10 digits, left padded with ascii '0')
  1001. */
  1002. xprintf("#+%010lu\n", (unsigned long)hp->Htime);
  1003. if (HistLit && hp->histline)
  1004. xprintf("%S\n", hp->histline);
  1005. else
  1006. prlex(&hp->Hlex);
  1007. cleanup_until(&old_output_raw);
  1008. }
  1009. else {
  1010. Char *cp = str2short("%h\t%T\t%R\n");
  1011. Char *p;
  1012. struct varent *vp = adrof(STRhistory);
  1013. if (vp && vp->vec != NULL && vp->vec[0] && vp->vec[1])
  1014. cp = vp->vec[1];
  1015. p = tprintf(FMT_HISTORY, cp, NULL, hp->Htime, hp);
  1016. cleanup_push(p, xfree);
  1017. for (cp = p; *cp;)
  1018. xputwchar(*cp++);
  1019. cleanup_until(p);
  1020. }
  1021. }
  1022. PG_STATIC void
  1023. dophist(int n, int hflg)
  1024. {
  1025. struct Hist *hp;
  1026. if (setintr) {
  1027. int old_pintr_disabled;
  1028. pintr_push_enable(&old_pintr_disabled);
  1029. cleanup_until(&old_pintr_disabled);
  1030. }
  1031. if ((hflg & HIST_REV) == 0) {
  1032. /* Since the history list is stored most recent first, non-reversing
  1033. * print needs to print (backwards) up the list. */
  1034. if ((unsigned)n >= histCount)
  1035. hp = histTail;
  1036. else {
  1037. for (hp = Histlist.Hnext;
  1038. --n > 0 && hp->Hnext != NULL;
  1039. hp = hp->Hnext)
  1040. ;
  1041. }
  1042. if (hp == NULL)
  1043. return; /* nothing to print */
  1044. for (; hp != &Histlist; hp = hp->Hprev)
  1045. phist(hp, hflg);
  1046. } else {
  1047. for (hp = Histlist.Hnext; n-- > 0 && hp != NULL; hp = hp->Hnext)
  1048. phist(hp, hflg);
  1049. }
  1050. }
  1051. /*ARGSUSED*/
  1052. void
  1053. dohist(Char **vp, struct command *c)
  1054. {
  1055. int n, hflg = 0;
  1056. USE(c);
  1057. if (getn(varval(STRhistory)) == 0)
  1058. return;
  1059. while (*++vp && **vp == '-') {
  1060. Char *vp2 = *vp;
  1061. while (*++vp2)
  1062. switch (*vp2) {
  1063. case 'c':
  1064. hflg |= HIST_CLEAR;
  1065. break;
  1066. case 'h':
  1067. hflg |= HIST_ONLY;
  1068. break;
  1069. case 'r':
  1070. hflg |= HIST_REV;
  1071. break;
  1072. case 'S':
  1073. hflg |= HIST_SAVE;
  1074. break;
  1075. case 'L':
  1076. hflg |= HIST_LOAD;
  1077. break;
  1078. case 'M':
  1079. hflg |= HIST_MERGE;
  1080. break;
  1081. case 'T':
  1082. hflg |= HIST_TIME;
  1083. break;
  1084. default:
  1085. stderror(ERR_HISTUS, "chrSLMT");
  1086. break;
  1087. }
  1088. }
  1089. if (hflg & HIST_CLEAR) {
  1090. struct Hist *np, *hp;
  1091. for (hp = &Histlist; (np = hp->Hnext) != NULL;)
  1092. hremove(np), hfree(np);
  1093. }
  1094. if (hflg & (HIST_LOAD | HIST_MERGE))
  1095. loadhist(*vp, (hflg & HIST_MERGE) ? 1 : 0);
  1096. else if (hflg & HIST_SAVE)
  1097. rechist(*vp, 1);
  1098. else {
  1099. if (*vp)
  1100. n = getn(*vp);
  1101. else {
  1102. n = getn(varval(STRhistory));
  1103. }
  1104. dophist(n, hflg);
  1105. }
  1106. }
  1107. char *
  1108. fmthist(int fmt, ptr_t ptr)
  1109. {
  1110. struct Hist *hp = ptr;
  1111. char *buf;
  1112. switch (fmt) {
  1113. case 'h':
  1114. return xasprintf("%6d", hp->Hnum);
  1115. case 'R':
  1116. if (HistLit && hp->histline)
  1117. return xasprintf("%S", hp->histline);
  1118. else {
  1119. Char *istr, *ip;
  1120. char *p;
  1121. istr = sprlex(&hp->Hlex);
  1122. buf = xmalloc(Strlen(istr) * MB_LEN_MAX + 1);
  1123. for (p = buf, ip = istr; *ip != '\0'; ip++)
  1124. p += one_wctomb(p, CHAR & *ip);
  1125. *p = '\0';
  1126. xfree(istr);
  1127. return buf;
  1128. }
  1129. default:
  1130. buf = xmalloc(1);
  1131. buf[0] = '\0';
  1132. return buf;
  1133. }
  1134. }
  1135. /* Save history before exiting the shell. */
  1136. void
  1137. rechist(Char *fname, int ref)
  1138. {
  1139. Char *snum;
  1140. int fp, ftmp, oldidfds;
  1141. struct varent *shist;
  1142. static Char *dumphist[] = {STRhistory, STRmhT, 0, 0};
  1143. if (fname == NULL && !ref)
  1144. return;
  1145. /*
  1146. * If $savehist is just set, we use the value of $history
  1147. * else we use the value in $savehist
  1148. */
  1149. if (((snum = varval(STRsavehist)) == STRNULL) &&
  1150. ((snum = varval(STRhistory)) == STRNULL))
  1151. snum = STRmaxint;
  1152. if (fname == NULL) {
  1153. if ((fname = varval(STRhistfile)) == STRNULL)
  1154. fname = Strspl(varval(STRhome), &STRtildothist[1]);
  1155. else
  1156. fname = Strsave(fname);
  1157. }
  1158. else
  1159. fname = globone(fname, G_ERROR);
  1160. cleanup_push(fname, xfree);
  1161. /*
  1162. * The 'savehist merge' feature is intended for an environment
  1163. * with numerous shells being in simultaneous use. Imagine
  1164. * any kind of window system. All these shells 'share' the same
  1165. * ~/.history file for recording their command line history.
  1166. * Currently the automatic merge can only succeed when the shells
  1167. * nicely quit one after another.
  1168. *
  1169. * Users that like to nuke their environment require here an atomic
  1170. * loadhist-creat-dohist(dumphist)-close
  1171. * sequence.
  1172. *
  1173. * jw.
  1174. */
  1175. /*
  1176. * We need the didfds stuff before loadhist otherwise
  1177. * exec in a script will fail to print if merge is set.
  1178. * From: mveksler@iil.intel.com (Veksler Michael)
  1179. */
  1180. oldidfds = didfds;
  1181. didfds = 0;
  1182. if ((shist = adrof(STRsavehist)) != NULL && shist->vec != NULL)
  1183. if (shist->vec[1] && eq(shist->vec[1], STRmerge))
  1184. loadhist(fname, 1);
  1185. fp = xcreat(short2str(fname), 0600);
  1186. cleanup_until(fname);
  1187. if (fp == -1) {
  1188. didfds = oldidfds;
  1189. return;
  1190. }
  1191. ftmp = SHOUT;
  1192. SHOUT = fp;
  1193. dumphist[2] = snum;
  1194. dohist(dumphist, NULL);
  1195. xclose(fp);
  1196. SHOUT = ftmp;
  1197. didfds = oldidfds;
  1198. }
  1199. /* This is the entry point for loading history data from a file. */
  1200. void
  1201. loadhist(Char *fname, int mflg)
  1202. {
  1203. static Char *loadhist_cmd[] = {STRsource, NULL, NULL, NULL};
  1204. loadhist_cmd[1] = mflg ? STRmm : STRmh;
  1205. if (fname != NULL)
  1206. loadhist_cmd[2] = fname;
  1207. else if ((fname = varval(STRhistfile)) != STRNULL)
  1208. loadhist_cmd[2] = fname;
  1209. else
  1210. loadhist_cmd[2] = STRtildothist;
  1211. dosource(loadhist_cmd, NULL);
  1212. /* During history merging (enthist sees mflg set), we disable management of
  1213. * Hnum and Href (because fastMergeErase is true). So now reset all the
  1214. * values based on the final ordering of the history list. */
  1215. if (mflg) {
  1216. int n = eventno;
  1217. struct Hist *hp = &Histlist;
  1218. while ((hp = hp->Hnext))
  1219. hp->Hnum = hp->Href = n--;
  1220. }
  1221. }