This program is a minimized version of a larger project that processes text files to generate CSV output with word frequencies. The full version works well for small to medium-sized files but encounters issues with very large texts like “The Lord of the Rings”.

The program must run on Unix-like systems and uses libraries exclusively of the C standard.

• It reads words from an input file, with a maximum word length of 30 characters.
• Words are sent through a pipe from a child process to the parent process.
• The parent process reads from the pipe and writes to an output file, formatting 100 words per line.
• The program uses basic file I/O, pipes, and process forking to demonstrate the core functionality.

1. I’ve simplified the original code to focus on the core functionality of reading, piping, and writing words.

2. I’ve implemented basic error checking for file operations and pipe creation.

3. I’ve used a fixed buffer size for reading words to avoid potential buffer overflow issues.

The program should successfully process files of any size, reading all words from the input file and writing them to the output file in the specified format (100 words per line). It should handle very large input files without memory issues or data loss.

While this minimized version may work for moderately sized files, I’m concerned about its performance and reliability with extremely large inputs. The original, more complex version struggles with very large files, and I’m trying to isolate the cause.

1. Are there any obvious issues in this code that could lead to problems with very large files?

2. How can I improve the pipe handling or memory management to better handle large amounts of data?

3. Are there alternative approaches or optimizations that would be more suitable for processing extremely large text files?

Below, I’ve included both the minimized code and the original, more extensive code. The minimized version attempts to emulate the core functionality where I believe the issue lies. For those who want to examine a more comprehensive implementation, please refer to the extended code below.

I’m not certain if I’ve successfully isolated the problem in this minimized version, so any insights on either the minimized or full code would be greatly appreciated.

#include <stdio.h>
#include <stdlib.h>
#include <string.h>
#include <unistd.h>
#include <sys/wait.h>

#define MAX_WORD_LENGTH 30
#define WORDS_PER_LINE 30

// Function to read words from file and write to pipe
void read_words(FILE *file, int pipe_fd) {
    char word[MAX_WORD_LENGTH + 1];
    int count = 0;

    while (fscanf(file, "%30s", word) == 1) {
        write(pipe_fd, word, strlen(word) + 1);
        count++;
        if (count % WORDS_PER_LINE == 0) {
            char newline = 'n';
            write(pipe_fd, &newline, 1);
        }
    }
    close(pipe_fd);
}

// Function to read words from pipe and write to output file
void write_words(int pipe_fd, FILE *output) {
    char word[MAX_WORD_LENGTH + 1];
    int bytes_read;
    int count = 0;

    while ((bytes_read = read(pipe_fd, word, MAX_WORD_LENGTH + 1)) > 0) {
        if (word[0] == 'n') {
            fprintf(output, "n");
            count = 0;
        } else {
            fprintf(output, "%s ", word);
            count++;
            if (count % WORDS_PER_LINE == 0) {
                fprintf(output, "n");
            }
        }
    }
    close(pipe_fd);
}

int main() {


    // Open input file
    FILE *input_file = fopen("input.txt", "r");
    if (!input_file) {
        perror("Failed to open input file");
        exit(1);
    }

    // Open output file
    FILE *output_file = fopen("output.txt", "w");
    if (!output_file) {
        perror("Failed to open output file");
        fclose(input_file);
        exit(1);
    }

    // Create pipe
    int pipefd[2];
    if (pipe(pipefd) == -1) {
        perror("pipe");
        exit(1);
    }

    // Fork process
    pid_t pid = fork();
    if (pid == -1) {
        perror("fork");
        exit(1);
    } else if (pid == 0) {  // Child process
        close(pipefd[0]);
        read_words(input_file, pipefd[1]);
        fclose(input_file);
        exit(0);
    } else {  // Parent process
        close(pipefd[1]);
        write_words(pipefd[0], output_file);
        fclose(output_file);
        wait(NULL);
    }

    return 0;
}

For a more detailed look at the original implementation, please refer to the following extended code:

#include <stdio.h>
#include <stdlib.h>
#include <string.h>
#include <stdbool.h>
#include <sys/types.h>
#include <sys/wait.h> 
#include <unistd.h>
#include <wchar.h>
#include <wctype.h>
#include <locale.h>

// Define a structure for an object containing a string, its frequency, and coordinates in the matrix
typedef struct {
    wchar_t String[31];  // The word or string, limited to 30 characters plus null terminator
    float frequency;     // The frequency of occurrence of this word
    int occurrence;      // The number of times this word appears
    int x;               // The x-coordinate in the matrix
    int y;               // The y-coordinate in the matrix
} WordObject;

// Define a structure for a matrix containing a list of WordObjects and its dimensions
typedef struct {
    WordObject** list;   // A dynamic 2D array of WordObjects
    int* x;              // An array to store the width of each row
    int y;               // The number of rows in the matrix
} Matrix;

// Define a structure for the alphabet containing sub-alphabets and a word
struct Alphabet {
    struct Alphabet* subAlphabet[46];  // Array of pointers to sub-alphabets (46 for extended character set)
    WordObject* word;                  // Pointer to a WordObject
};

// Define the Dictionary type as a pointer to struct Alphabet
typedef struct Alphabet Dictionary;

// Function declarations
Dictionary* deallocation(Dictionary* dict);
int createTable(FILE* file, Matrix* matrix, Dictionary* dict, int pipe1[], int pipe2[], wchar_t firstWord[]);
int asciiIndex(wchar_t character);
bool compareStrings(wchar_t* string1, wchar_t* string2);
bool operativeSearch(wchar_t string[], int index, Dictionary* dict, int i, int p, Matrix* m);
void readWordFromInputFile(FILE* file, int pipe1[], long int position);
void print(FILE* sheet, int pipe2[]);
long int firstWord(FILE* file, Matrix* matrix);

// Main function
int main(void) {
    // Set the locale to support Unicode
    setlocale(LC_ALL, "en_US.UTF-8");

    // Open input and output files
    FILE* inputFile = fopen("input.txt", "r");
    FILE* outputFile = fopen("output.csv", "w");

    // Check if file opening was successful
    if (inputFile == NULL || outputFile == NULL) {
        exit(EXIT_FAILURE);  // Exit with error if file opening failed
    }

    // Allocate memory for the dictionary
    Dictionary* dictionary = (Dictionary*)calloc(1, sizeof(Dictionary));
    dictionary->word = (WordObject*)calloc(1, sizeof(WordObject));

    // Initialize the matrix
    Matrix matrix;
    matrix.list = (WordObject**)calloc(1, sizeof(WordObject*));
    matrix.list[0] = (WordObject*)calloc(2, sizeof(WordObject));
    matrix.x = (int*)calloc(1, sizeof(int));
    matrix.x[0] = 1;

    // Find the position of the first word in the input file
    long int position = firstWord(inputFile, &matrix);
    if (position == -1) {
        exit(EXIT_FAILURE);  // Exit with error if no valid word was found
    }

    // Create two pipes for inter-process communication
    int pipe1[2], pipe2[2];
    if (pipe(pipe1) == -1 || pipe(pipe2) == -1) {
        exit(EXIT_FAILURE);  // Exit with error if pipe creation failed
    }

    // Variable to store the maximum number of bytes in a matrix row
    int MAX;

    // Declare process IDs for the three child processes
    pid_t pid_Process1, pid_Process2, pid_Process3;

    // Create the first child process
    pid_Process1 = fork();
    if (pid_Process1 == -1) {
        // If fork fails, print error message and exit
        perror("Fork error");
        return 1;
    } else if (pid_Process1 == 0) {
        // Child process 1
        close(pipe1[0]);  // Close read end of pipe1
        // Read words from input file and write to pipe1
        readWordFromInputFile(inputFile, pipe1, position);
        exit(0);  // Exit child process
    } else {
        // Parent process
        close(pipe1[1]);  // Close write end of pipe1
        waitpid(pid_Process1, NULL, 0);  // Wait for child process 1 to finish
    }

    // Create the second child process
    pid_Process2 = fork();
    if (pid_Process2 == -1) {
        perror("Fork error");
        return 1;
    } else if (pid_Process2 == 0) {
        // Child process 2
        close(pipe2[0]);  // Close read end of pipe2
        wchar_t firstWordArray[31];
        wcscpy(firstWordArray, matrix.list[0][0].String);
        // Create table and write to pipe2
        createTable(inputFile, &matrix, dictionary, pipe1, pipe2, firstWordArray);
        exit(0);
    } else {
        // Parent process
        close(pipe1[0]);  // Close read end of pipe1
        close(pipe2[1]);  // Close write end of pipe2
        waitpid(pid_Process2, NULL, 0);  // Wait for child process 2 to finish
    }

    // Create the third child process
    pid_Process3 = fork();
    if (pid_Process3 == -1) {
        perror("Fork error");
        return 1;
    } else if (pid_Process3 == 0) {
        // Child process 3
        close(pipe2[1]);  // Close write end of pipe2
        // Read from pipe2 and print to output file
        print(outputFile, pipe2);
        exit(0);
    }

    // Close all pipe ends in the parent process
    close(pipe1[0]);
    close(pipe1[1]);
    close(pipe2[0]);
    close(pipe2[1]);

    // Wait for all child processes to finish
    waitpid(pid_Process3, NULL, 0);
    wait(NULL);
    wait(NULL);
    wait(NULL);

    // Close input and output files
    fclose(outputFile);
    fclose(inputFile);

    // Free allocated memory for the dictionary
    for (int i = 0; i < 46; i++) {
        dictionary->subAlphabet[i] = deallocation(dictionary->subAlphabet[i]);
    }
    free(dictionary->word);
    free(dictionary);

    return 0;  // Exit successfully
}
// Function to deallocate memory for the Dictionary structure
Dictionary* deallocation(Dictionary* dict) {
    // If the dictionary is NULL, return NULL
    if (dict == NULL) {
        return NULL;
    }

    // Recursively deallocate memory for all sub-alphabets
    for (int i = 0; i < 46; i++) {
        dict->subAlphabet[i] = deallocation(dict->subAlphabet[i]);
    }

    // Free the memory allocated for the word
    free(dict->word);

    // Free the memory allocated for the dictionary itself
    free(dict);

    return NULL;
}

// Function to get the ASCII index of a character
int asciiIndex(wchar_t character) {
    // Convert the character to lowercase
    wchar_t c = towlower(character);

    if (iswalpha(c)) {
        // Handle non-accented letters
        return c - L'a';
    } else {
        // Handle accented characters and special symbols
        switch (c) {
            case L'à': return 26;
            case L'è': return 27;
            case L'é': return 28;
            case L'ì': return 29;
            case L'ò': return 30;
            case L'ù': return 31;
            case L'!': return 32;
            case L'?': return 33;
            case L'.': return 34;
            case L''': return 35;
        }
    }

    // Handle numbers
    if (c >= L'0' && c <= L'9') {
        return 36 + (c - L'0');
    }

    // Return -1 for unrecognized characters
    return -1;
}

// Function to compare two strings
bool compareStrings(wchar_t* String1, wchar_t* String2) {
    // If both strings are empty, they are equal
    if (*String1 == L'' && *String2 == L'') {
        return true;
    }
    
    // If the characters are the same or the same when converted to lowercase
    if (*String1 == *String2 || (towlower((wchar_t)*String1) == towlower((wchar_t)*String2))) {
        // Recursively compare the next characters
        return compareStrings(String1 + 1, String2 + 1);
    }

    // If the characters are different, the strings are not equal
    return false;
}
// Function to find the first word in the input file
long int firstWord(FILE* file, Matrix* matrix) {
    wchar_t character;
    const wchar_t* accentedCharacters = L"àèéìòóùÀÈÉÌÒÓÙ";
    int index = 0;
    long int pos = 0;

    while ((character = fgetwc(file)) != WEOF) {
        wchar_t next = fgetwc(file);
        ungetwc(next, file);

        // Skip initial newlines or spaces
        if (character == L'n' || character == L' ') {
            continue;
        } else if (iswalnum(character) || (wcschr(accentedCharacters, character) != NULL)) {
            // Add alphanumeric or accented characters to the matrix
            matrix->list[0][0].String[index] = character;
            index++;

            // Check if this is the end of the word
            if (next == L' ' || (next == L'n' || next == L'r') || next == L'!' || next == L'?' || next == L'.') {
                matrix->list[0][0].String[index] = L'';
                index = 0;
                return ftell(file);
            } else if (next == L''') {
                // Handle apostrophes
                matrix->list[0][0].String[index] = L''';
                matrix->list[0][0].String[index + 1] = L'';
                index = 0;
                return ftell(file);
            }
            continue;
        } else if (character == L'!' || character == L'?' || character == L'.') {
            // Handle punctuation marks
            index = 0;
            matrix->list[0][0].String[index++] = character;
            if (next == ''') {
                matrix->list[0][0].String[index++] = ''';
                fgetwc(file);
            }
            matrix->list[0][0].String[index] = L'';
            index = 0;
            return ftell(file);
        }
    }
    
    // Return -1 if no valid word was found
    return -1;
}

// Function for operative search of a string in the matrix
bool operativeSearch(wchar_t string[], int index, Dictionary* dict, int i, int p, Matrix* m) {
    Dictionary* currentDict = dict;
    int ASCII;

    // If we've reached the end of the string
    if (string[index] == '') {
        // Check if the string matches the word in the current dictionary node
        if (compareStrings(currentDict->word->String, m->list[i][0].String)) {
            int n = 1;
            int y = currentDict->word->y;
            int x = currentDict->word->x;
            int found = 0;

            // Increment the occurrence count
            m->list[y][0].occurrence++;

            // Update frequencies for existing words
            while (n <= m->x[y] && m->list[y][n].x == n) {
                if (compareStrings(m->list[y][n].String, m->list[i][p].String)) {
                    m->list[y][n].occurrence++;
                    m->list[y][n].frequency = (float)m->list[y][n].occurrence / (float)m->list[y][0].occurrence;
                    found = 1;
                }
                m->list[y][n].frequency = (float)m->list[y][n].occurrence / (float)m->list[y][0].occurrence;
                n++;
            }

            // If the word wasn't found, add it
            if (!found) {
                if (n > m->x[y]) {
                    m->x[y] = n;
                }
                m->list[y] = realloc(m->list[y], (n + 1) * sizeof(WordObject));
                wcscpy(m->list[y][n].String, m->list[i][p].String);
                m->list[y][n].occurrence = 1;
                m->list[y][n].frequency = (float)m->list[y][n].occurrence / (float)m->list[y][0].occurrence;
                m->list[y][n].x = n;
            }

            return true;
        }

        // If no match was found, add the new word to the dictionary
        wcscpy(currentDict->word->String, string);
        currentDict->word->x = p;
        currentDict->word->y = i;
        m->list[i][1].x = 1;
        m->list[i][0].x = 0;
        m->list[i][1].occurrence = 1;
        m->list[i][0].occurrence = 1;
        m->list[i][1].frequency = (float)m->list[i][1].occurrence / (float)m->list[i][0].occurrence;

        return false;
    }

    // Get the ASCII index for the current character
    ASCII = asciiIndex(string[index]);

    // If the sub-alphabet for this character doesn't exist, create it
    if (currentDict->subAlphabet[ASCII] == NULL) {
        currentDict->subAlphabet[ASCII] = (Dictionary*)calloc(1, sizeof(Dictionary));
        if (currentDict->subAlphabet[ASCII] == NULL) {
            exit(1);
        }
        currentDict->subAlphabet[ASCII]->word = (WordObject*)calloc(1, sizeof(WordObject));
        if (currentDict->subAlphabet[ASCII]->word == NULL) {
            exit(1);
        }
    }

    // Move to the next sub-alphabet
    currentDict = currentDict->subAlphabet[ASCII];

    // Recursive call for the next character
    return operativeSearch(string, index + 1, currentDict, i, p, m);
}
// Function to create the table from the input
int createTable(FILE* file, Matrix* matrix, Dictionary* dict, int pipe1[], int pipe2[], wchar_t firstWord[]) {
    wchar_t string1[31];  // Buffer to store each word (max 30 characters + null terminator)
    matrix->y = 1;  // Initialize the number of rows to 1
    matrix->list = NULL;  // Initialize the list to NULL

    // Allocate memory for the first row of the matrix
    matrix->list = (WordObject**)calloc(1, matrix->y * sizeof(WordObject*));
    matrix->list[matrix->y - 1] = (WordObject*)calloc(1, 2 * sizeof(WordObject));

    // Copy the first word into the matrix
    wcscpy(matrix->list[0][0].String, firstWord);

    // Read all words from pipe1
    while (read(pipe1[0], &string1, sizeof(string1)) > 0) {
        if (string1[0] == '') {  // If an empty string is read, break the loop
            break;
        }

        // Add the word to the matrix
        wcscpy(matrix->list[matrix->y - 1][1].String, string1);
        matrix->list[matrix->y - 1][1].x = 1;
        matrix->list[matrix->y - 1][0].x = 0;

        // Perform operative search and update the matrix
        int control = operativeSearch(matrix->list[matrix->y - 1][0].String, 0, dict, matrix->y - 1, 1, matrix);
        if (control == 0) {
            // If a new word is found, expand the matrix
            matrix->y++;
            matrix->list = realloc(matrix->list, matrix->y * sizeof(WordObject*));
            matrix->list[matrix->y - 1] = (WordObject*)calloc(2, sizeof(WordObject));
            matrix->x = realloc(matrix->x, matrix->y * sizeof(int));
            matrix->x[matrix->y - 1] = 1;
        }

        // Copy the current word to be the first word of the next row
        wcscpy(matrix->list[matrix->y - 1][0].String, string1);
        string1[0] = '';  // Reset the string
    }

    // Handle the last word
    wcscpy(matrix->list[matrix->y - 1][1].String, matrix->list[0][0].String);
    int control = operativeSearch(matrix->list[matrix->y - 1][0].String, 0, dict, matrix->y - 1, 1, matrix);
    if (control == 0) {
        matrix->y++;
    }
    close(pipe1[0]);  // Close pipe1

    // Write the number of rows to pipe2
    write(pipe2[1], &(matrix->y), sizeof(int));

    // Write each row of the matrix to pipe2
    for (int i = 0; i < matrix->y-1; i++) {
        // Calculate the length of the row
        size_t len = wcslen(matrix->list[i][0].String);
        for (int j = 1; j <= matrix->x[i]; j++) {
            len += wcslen(matrix->list[i][j].String) + 7;  // +7 for ",%.4f,"
        }
        len++;  // For the newline character

        // Allocate memory for the row
        wchar_t* row = calloc(len + 1, sizeof(wchar_t));
        
        // Construct the row string
        swprintf(row, len + 1, L"%ls", matrix->list[i][0].String);
        for (int j = 1; j <= matrix->x[i]; j++) {
            wchar_t temp[50];
            swprintf(temp, 50, L",%ls,%.4f", matrix->list[i][j].String, matrix->list[i][j].frequency);
            len += 50;
            row = realloc(row, len * sizeof(wchar_t));
            wcscat(row, temp);
        }
        wcscat(row, L"n");

        // Write the length of the row and the row itself to pipe2
        write(pipe2[1], &len, sizeof(size_t));
        write(pipe2[1], row, len * sizeof(wchar_t));

        free(row);
    }

    // Write end signal (length 0) to pipe2
    size_t end_signal = 0;
    write(pipe2[1], &end_signal, sizeof(size_t));

    close(pipe2[1]);
    return 0;
}
// Function to print the table to the output file
void print(FILE* outputFile, int pipe2[]) {
    close(pipe2[1]);  // Close the write end of pipe2

    int numRows;
    ssize_t bytesRead = read(pipe2[0], &numRows, sizeof(int));
    if (bytesRead != sizeof(int)) {
        if (bytesRead == 0) {
            fprintf(stderr, "Pipe closed unexpectedly while reading the number of rowsn");
        } else {
            perror("Error reading the number of rows");
        }
        exit(EXIT_FAILURE);
    }
    fprintf(stderr, "Number of rows read: %dn", numRows);

    for (int i = 0; i < numRows; i++) {
        size_t len;
        bytesRead = read(pipe2[0], &len, sizeof(size_t));
        if (bytesRead != sizeof(size_t)) {
            if (bytesRead == 0) {
                fprintf(stderr, "Pipe closed unexpectedly while reading the length of row %dn", i);
            } else {
                perror("Error reading the row length");
            }
            exit(EXIT_FAILURE);
        }
        fprintf(stderr, "Length of row %d: %zun", i, len);

        if (len == 0) {
            fprintf(stderr, "Received end of data signaln");
            break;
        }

        wchar_t* row = calloc(len + 1, sizeof(wchar_t));
        if (row == NULL) {
            perror("Error allocating memory for the row");
            exit(EXIT_FAILURE);
        }

        bytesRead = read(pipe2[0], row, len * sizeof(wchar_t));
        if (bytesRead != len * sizeof(wchar_t)) {
            if (bytesRead == 0) {
                fprintf(stderr, "Pipe closed unexpectedly while reading row %dn", i);
            } else {
                perror("Error reading the row");
            }
            free(row);
            exit(EXIT_FAILURE);
        }

        row[len] = L'';
        fprintf(stderr, "Row %d read: %lsn", i, row);
        fwprintf(outputFile, L"%ls", row);

        free(row);
    }

    close(pipe2[0]);
    fprintf(stderr, "Printing completedn");
}

// Function to read words from the input file
void readWordFromInputFile(FILE* file, int pipe1[], long int position) {
    wchar_t string1[31];
    int index = 0;
    wchar_t character;
    const wchar_t* accentedCharacters = L"àèéìòóùÀÈÉÌÒÓÙ";
    
    // Position the file pointer
    for(int i = 0; i == position; i++){
        fgetwc(file);
    }

    while ((character = fgetwc(file)) != WEOF) {
        wchar_t next = fgetwc(file);
        ungetwc(next, file);

        // Handle end of file
        if ((character == L' ' || (character == L'n' || character == L'r')) && next == WEOF) {
            if (index > 0) {
                string1[index] = L'';
                write(pipe1[1], string1, sizeof(string1));
            }
            break;
        }

        // Handle punctuation
        if (character == L'!' || character == L'?' || character == L'.') {
            if (index > 0) {
                string1[index] = L'';
                write(pipe1[1], string1, sizeof(string1));
                index = 0;
            }
            string1[index] = character;
            if (next == L''') {
                index++;
                string1[index] = L''';
                fgetwc(file);
            }
            index++;
            if (next == WEOF) {
                string1[index] = L'';
                write(pipe1[1], string1, sizeof(string1));
                break;
            }
        } else if (index > 0 && character == ''') {
            string1[index] = L''';
            index++;
            string1[index] = L'';
            write(pipe1[1], string1, sizeof(string1));
            index = 0;
            string1[index] = L'';
        } else if (iswalnum(character) || (wcschr(accentedCharacters, character) != NULL)) {
            if (index > 0 && (string1[index - 1] == '!' || string1[index - 1] == '?' || string1[index - 1] == '.' || string1[index - 1] == ''')) {
                string1[index] = L'';
                write(pipe1[1], string1, sizeof(string1));
                index = 0;
                string1[index] = L'';
            }
            string1[index++] = character;
            if (next == ''') {
                string1[index++] = L''';
                string1[index] = L'';
                write(pipe1[1], string1, sizeof(string1));
                fgetwc(file);
                index = 0;
            }
            if (next == L' ' || (next == L'n' || next == L'r') || next == WEOF ||
                next == L'!' || next == L'?' || next == L'.') {
                string1[index] = L'';
                write(pipe1[1], string1, sizeof(string1));
                index = 0;
            }
        } else if (character == L' ' || (character == L'n' || character == L'r')) {
            if (index > 0) {
                string1[index] = L'';
                write(pipe1[1], string1, sizeof(string1));
                index = 0;
                string1[index] = L'';
            }
        } else if (!(iswalnum(character) || (wcschr(accentedCharacters, character) != NULL) ||
                     character == L''' || character == L'!' ||
                     character == L'?' || character == L'.') &&
                   (next == L' ' || (next == L'n' || next == L'r'))) {
            continue;
        }

        if (next == WEOF) {
            string1[index] = L'';
            write(pipe1[1], string1, sizeof(string1));
            break;
        }
    }

    close(pipe1[1]);
}