Examples

Create frequency graph from a log

Task: Read /var/log/dpkg.log and create a graph to visualize how often packages are installed, upgraded and removed.

Solution: The loop (91) calls function read_log which reads the log line by line (33) and splits the fields date and time in minutes (35,39) and status (40,42). Third field of the log status is status of the dpkg operation(install, upgrade, remove …). Command ZINCRBY (43) increments by 1 the score of date in the key status. As a result the database contains keys(install, upgrade, remove …) and associated lists of dates sorted by score. Next loop (97) calls the function write_csv with first 10 keys. As a result status.csv files are created in the current directory with the (date;score) pairs.

[create-graph-01.c]

  1
  2
  3
  4
  5
  6
  7
  8
  9
 10
 11
 12
 13
 14
 15
 16
 17
 18
 19
 20
 21
 22
 23
 24
 25
 26
 27
 28
 29
 30
 31
 32
 33
 34
 35
 36
 37
 38
 39
 40
 41
 42
 43
 44
 45
 46
 47
 48
 49
 50
 51
 52
 53
 54
 55
 56
 57
 58
 59
 60
 61
 62
 63
 64
 65
 66
 67
 68
 69
 70
 71
 72
 73
 74
 75
 76
 77
 78
 79
 80
 81
 82
 83
 84
 85
 86
 87
 88
 89
 90
 91
 92
 93
 94
 95
 96
 97
 98
 99
100
101
102
103
104
/* 
 * Task: Read /var/log/dpkg.log and create a graph to visualize how
 * often packages are installed, upgraded and removed.
 * Tested with: gcc 7.2, hiredis 0.13 and redis 4.0.1
 */

#include <assert.h>
#include <stdio.h>
#include <stdlib.h>
#include <string.h>
#include <hiredis.h>

#define RCMD(context, ...) {                  \
  freeReplyObject(reply);                     \
  reply = redisCommand(context, __VA_ARGS__); \
  assert(reply != NULL);                      \
  }

const char LOG_SEPARATOR = ' ';
const char CSV_SEPARATOR = ';';
const char *LOG_FILES[] = { "dpkg.log", };

redisContext *c;

/* This function reads log_file and puts the status into the database.*/
int read_log(const char *log_file, redisContext *c) {
  redisReply *reply=NULL;
  FILE *fp;
  char line[256];
  char *p, *date, *status;

  fp = fopen(log_file, "r");
  while (fgets(line, sizeof(line), fp) != NULL) {
    p = line;
    date = p++;
    while (p[0] != LOG_SEPARATOR) p++;
    p++;
    while (p[0] != LOG_SEPARATOR) p++;
    *(p-3) = '\0';
    status = ++p;
    while (p[0] != LOG_SEPARATOR) p++;
    *p = '\0';
    RCMD(c, "ZINCRBY %s 1 \"%s\"", status, date);
  }
  fclose(fp);
  return(0);
}

/* This function reads the database and writes the status CSV file. */
int write_csv(char *status, redisContext *c) {
  redisReply *reply=NULL;
  FILE *fp;
  int i,n;
  char filename[256];

  strcpy(filename, status);
  strcat(filename, ".csv");
  fp = fopen(filename, "w");
  RCMD(c, "ZRANGE %s 0 -1 WITHSCORES", status);
  n = reply->elements-1;
  for (i=0; i<n; i=i+2) {
    fprintf(fp, "%s %c %s\n", reply->element[i]->str, \
	    CSV_SEPARATOR, \
	    reply->element[i+1]->str);
  }
  fclose(fp);
  return(0);
}


int main(int argc, char **argv) {
  redisReply *reply=NULL;
  int i, n;
  char *status;
  
  c = redisConnect("localhost", 6379);
  if (c == NULL || c->err) {
    if (c) {
      printf("Connection error: %s\n", c->errstr);
      redisFree(c);
    } else {
      printf("Connection error: can't allocate redis context\n");
    }
    exit(1);
  }
	
  RCMD(c, "SELECT 0");
  RCMD(c, "FLUSHDB");

  n = sizeof(LOG_FILES)/sizeof(LOG_FILES[0]);
  for (i = 0; i < n; i++) {
    read_log(LOG_FILES[i], c);
  }

  RCMD(c, "SCAN 0 COUNT 10");
  n = reply->element[1]->elements;
  for (i=0; i<n; i++) {
    status = reply->element[1]->element[i]->str;
    write_csv(status, c);
  }

  redisFree(c);
  return(0);
}

Result: The status.csv files can be used to create a graph with gnuplot.

[create-graph-01.gnuplot]

_images/graph-01.png

List 10 most used words in a text

Task: Read text from a file and list 10 most frequently used words in it.

Solution: Let’s use article about Redis at wikipedia.org as a text.

[create-topchart-text.bash]

#!/bin/bash
lynx -dump -nolist https://en.wikipedia.org/wiki/Redis > redis.txt

Command ZINCRBY (50) increments by 1 the score of word in the key topchart and ZRANGE (57) returns top 10 words with scores.

[create-topchart.c]

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
/* 
 * Task: Read text from a file and list 10 most frequently used words
 * in it.
 * Tested with: gcc 7.2, hiredis 0.13 and redis 4.0.1
 */

#include <assert.h>
#include <stdio.h>
#include <stdlib.h>
#include <string.h>
#include <hiredis.h>

#define RCMD(context, ...) {                  \
  freeReplyObject(reply);                     \
  reply = redisCommand(context, __VA_ARGS__); \
  assert(reply != NULL);                      \
  }

const int SIZEOFLINE = 1023;
const char* DELIMETERS = " ;,.-!?";
const char* filename = "redis.txt";

int main(int argc, char **argv) {
  redisContext *c;
  redisReply *reply=NULL;
  FILE *fp;
  char line[SIZEOFLINE+1];
  char* word;
  int i, n;
  
  c = redisConnect("localhost", 6379);
  if (c == NULL || c->err) {
    if (c) {
      printf("Connection error: %s\n", c->errstr);
      redisFree(c);
    } else {
      printf("Connection error: can't allocate redis context\n");
    }
    exit(1);
  }
	
  RCMD(c, "SELECT 0");
  RCMD(c, "FLUSHDB");

  fp = fopen(filename, "r");
  while (fgets(line, SIZEOFLINE, fp) != NULL) {
    word = strtok(line, DELIMETERS);
    while (word != NULL) {
      if (strlen(word) > 1) {
	RCMD(c, "ZINCRBY \"topchart\" 1 \"%s\"", word);
      }
      word = strtok (NULL, DELIMETERS);
    }
  }
  fclose(fp);

  RCMD(c, "ZRANGE \"topchart\" -10 -1 WITHSCORES");
  n = reply->elements-1;
  for (i=0; i<n; i=i+2) {
    printf("%s %s\n", reply->element[i+1]->str, reply->element[i]->str);
  }
  
  redisFree(c);
  return(0);
}

Result:

> ./create-topchart
10 "Retrieved"
10 "by"
10 "database"
19 "in"
21 "is"
24 "and"
30 "of"
30 "to"
33 "the"
57 "Redis"