Storing compressed data in MongoDB and decompressing it is a necessary task when you store large data files like Logs.

Database schema

export const DataSchema = new mongoose.Schema({
    clientId: String,
    data: Buffer,
    from: Number,
    to: Number,
    createdAt: {
        type: Number,
        default: Date.now()
    },    
})

Interface

export interface Data {
    clientId: string,
    data: Buffer,
    from: number,
    to: number,
    createdAt: number
}

Upload Controller

Method: POST – ‘multipart/form-data

Field name: file

Type: Use .zip file for lightweight

Allows to use multiple text files in a zip file

Upload Service

Input: file

Extract the zip file. In my case, use npm AdmZip to extract zip file, and zlib to compress data before insert to database (my case is mongodb):

// text logs is a json like as: {"time":"2021-12-01 08:08:26", "message":"Something", "data":"any text or object"}
// buffer: is your file data
const zlib = require('node:zlib');
const admZip = require("adm-zip");
let zip = new admZip(buffer)
let zipEntries: any[] = zip.getEntries(); // an array of ZipEntry records
let documents: any[] = [] // document list to insert DB
let i = 0
async function loop() {
    let zipEntry = zipEntries[i]
    let string = zipEntry.getData().toString("utf8");
    let array = string.split("\n");
    let data: any[] = []; //
    array.forEach(element => {
        if (element.length > 0) {
            try {
                let obj: any = JSON.parse(element);
                if (typeof obj === "object") {
                    obj.time = new Date(obj.time).valueOf();
                    data.push(obj);
                }
                // console.log(obj) 
            } catch (error) {
                console.log(error.message);
            }
        }
    });
    if (data.length > 0) {
        document.from = data[0].time as number;
        document.to = data[data.length - 1].time as number;
        document.data =  zlib.deflateSync(Buffer.from(JSON.stringify(data), "utf-8")); 
        documents.push(document);
    }
    if (i < zipEntries.length - 1) {
        i++
        await loop()
    }
}
await loop()
console.log(documents)
if (documents.length > 0) {
    await DataModel.insertMany(documents, { ordered: false }).then(res => {
        console.log(res)
    }).catch(error => {
        console.log(error)
    })
}

Search Controller

Require params:

  • clientId: string
  • from: number
  • to: number

Search Service

Query your data then extract data.

let documents = DataModel.find({
    clientId: params.clientId,
    from: { $lte: params.to }, 
    to: { $gte: params.from },         

})
if (documents.length > 0) {
    console.log("documents.length = " + documents.length)
    let i = 0
    async function loop() {   
        let buf = Buffer.from(documents[i].data.toString("hex"), "hex")       
        let json = (zlib.inflateSync(data)).toString("utf-8")
        let arrObj: RawData[] = JSON.parse(json)
        for (let j = 0; j < arrObj.length; j++) {
            if (arrObj[j].time && typeof arrObj[j].time === "number" && (arrObj[j].time as number) >= (params.from as number) && (arrObj[j].time as number) <= (params.to as number)) {
                queryObj.push(arrObj[j])
            }
        }
        if (i < documents.length - 1) {
            i++
            await loop()
        }
    }
    await loop()
}
if (queryObj.length > 0) {
    queryObj.sort((a, b) => (a.time as number) - (b.time as number))
    for (let i = 1; i < queryObj.length; i++) {
        if (queryObj[i].time === queryObj[i - 1].time && queryObj[i].data === queryObj[i - 1].data) {
            queryObj.splice(i, 1)
            i--
        }
    }
    queryObj.flatMap((element) => {
        element.time = new Date(element.time).toLocaleString("vi")
    })
    return queryObj


Leave a Reply

Your email address will not be published. Required fields are marked *