/*****************************************************************************
Welcome to GDB Online.
GDB online is an online compiler and debugger tool for C, C++, Python, Java,
PHP, Ruby, Perl, C#, VB, Swift, Pascal, Fortran, Haskell, Objective-C, Assembly,
HTML, CSS, JS, SQLite, Prolog. Code, Compile, Run and Debug online from anywhere
in world.

*******************************************************************************/
#include <stdio.h>
#include <stdlib.h>
#include <string.h>
#include <sys/stat.h>
#include <sys/types.h>

size_t
getFilesize (const char *filename)
{
  struct stat st;
  stat (filename, &st);
  return st.st_size;
}

void parsedata (FILE ** fp2, char **buffer);

int
main ()
{
  unsigned long intsize = 4194304;
  char *buffer = malloc (intsize);
  buffer[intsize - 1] = '\0';
  unsigned long cnt = 0;
  unsigned long seekoff = 0;
  unsigned long tiles = 0;
  unsigned long fsize = getFilesize ("list.txt");
  FILE *fp = fopen ("list.txt", "rb");
  FILE *fp2 = fopen ("nodup.txt", "w");
  setvbuf ( fp2 , NULL , _IOFBF , 4194304);
  while (fread (buffer, intsize - 1, 1, fp))
    {
      cnt++;
      seekoff = ftell (fp);
      int nlpos = intsize - 1UL;
      while (buffer[nlpos] != '\n')
	{
	  nlpos--;
	  seekoff = seekoff - 1UL;
	}
      fseek (fp, seekoff, SEEK_SET);
      buffer[nlpos - 1] = '\0';
      buffer[intsize] = '\0';
      parsedata (&fp2, &buffer);
      tiles = ftell (fp);
    }
    
  unsigned long lastblock = fsize - tiles;
  buffer[lastblock - 1] = '\0';
  fread (buffer, lastblock + 1, 1, fp);
  parsedata (&fp2, &buffer);
  fprintf (fp2, "%c", '\n');
 fflush (fp2);
  fclose (fp);
  fclose (fp2);
}

void
parsedata (FILE ** fp2, char **buffer)
{
  int felder = 45;
  int ignore = 3;
  char *cols[felder];
  char *backup = NULL;
  int n = 0;
  unsigned long seps = 0;
  char *last = NULL;
  int first = 1;
  for (char *p = strtok_r (*buffer, ";", &backup); p != NULL;
       p = strtok_r (NULL, ";", &backup))
    {
      seps = p - last;
      if (first == 1)
	first = 0;
      else
	while (seps > 0)
	  {
	    seps--;
	    cols[n++] = "";
	    fprintf (*fp2, ";");
	    if (n > felder - 1)
		n = 0;
	  }
      if (n < ignore + 1)
	fprintf (*fp2, "%s;", p);
      else if (n > ignore && n < felder + 1)
	{
	  for (int x = n - 1; x > ignore - 1; x--)
	    {
	      if (strcmp (cols[x], p) == 0)
		  break;
	      else if (x < ignore + 1)
		fprintf (*fp2, "%s", p);
	    }
	  fprintf (*fp2, ";");
	}
        if (n > felder -1)
	{
	n = 0;
     }
    cols[n++] = p;
    last = backup;
    }
     while (n < felder)
    {
	fprintf (*fp2, ";");
	n++;
    }
}