FlatFile.Delimited.Attributes 2.0.1

FlatFile

FlatFile is a library to work with flat files (work up-to 100 times faster then FileHelpers)

Modernization status

• 🚨 v2 breaking change: dropped legacy .NET Framework targets (net35-net48) and old build pipeline.
• ✅ Modernized runtime support to .NET 8 only via SDK-style projects.
• ✅ CI now builds modern projects with dotnet build on GitHub Actions.

Modern .NET support

Active projects:

• src/FlatFile.Core.Modern
• src/FlatFile.Core.Attributes.Modern
• src/FlatFile.Delimited.Modern
• src/FlatFile.FixedLength.Modern
• src/FlatFile.Delimited.Attributes.Modern
• src/FlatFile.FixedLength.Attributes.Modern

All of them target net8.0 and carry package/assembly version 2.0.0.

NuGet publishing from GitHub

When changes are merged to master, GitHub Actions can publish v2 packages automatically using .github/workflows/publish-nuget.yml.

Required repository secret:

• NUGET_API_KEY: NuGet.org API key with push permission for FlatFile packages.

The publish workflow packs all *.Modern projects and pushes resulting .nupkg files to NuGet (--skip-duplicate).

Installing FlatFile

Installing all packages

You should install FlatFile with NuGet:

Install-Package FlatFile

Installing FlatFile.Delimited

You should install FlatFile.Delimited with NuGet:

Install-Package FlatFile.Delimited

Add attribute-mapping extensions

You should install FlatFile.Delimited.Attributes with NuGet:

Install-Package FlatFile.Delimited.Attributes

Installing FlatFile.FixedLength

You should install FlatFile.FixedLength with NuGet:

Install-Package FlatFile.FixedLength

Add attribute-mapping extensions

You should install FlatFile.FixedLength.Attributes with NuGet:

Install-Package FlatFile.FixedLength.Attributes

This commands from Package Manager Console will download and install FlatFile and all required dependencies.

Benchmarks

Simple write

Simple read

Big (100000 entities) write

Big (100000 entities) write with reflection magic

FlatFile vs CsvHelper

Write all records with class mapping

Read all records with class mapping

Usage

Class mapping

DelimitedLayout

public sealed class DelimitedSampleRecordLayout : DelimitedLayout<FixedSampleRecord>
{
    public DelimitedSampleRecordLayout()
    {
        this.WithDelimiter(";")
            .WithQuote("\"")
            .WithMember(x => x.Cuit)
            .WithMember(x => x.Nombre)
            .WithMember(x => x.Actividad, c => c.WithName("AnotherName"));
    }
}

FixedLayout

public sealed class FixedSampleRecordLayout : FixedLayout<FixedSampleRecord>
{
    public FixedSampleRecordLayout()
    {
        this.WithMember(x => x.Cuit, c => c.WithLength(11))
            .WithMember(x => x.Nombre, c => c.WithLength(160))
            .WithMember(x => x.Actividad, c => c.WithLength(6));
    }
}

Run-time mapping

DelimitedLayout

public class LayoutFactory
{
    public IDelimitedLayout<TestObject> GetLayout()
    {
        IDelimitedLayout<TestObject> layout = new DelimitedLayout<TestObject>()
            .WithDelimiter(";")
            .WithQuote("\"")
            .WithMember(o => o.Id)
            .WithMember(o => o.Description)
            .WithMember(o => o.NullableInt, set => set.AllowNull("=Null"));

        return layout;
    } 
}

FixedLayout

public class LayoutFactory
{
    public IFixedLayout<TestObject> GetLayout()
    {
        IFixedLayout<TestObject> layout = new FixedLayout<TestObject>()
            .WithMember(o => o.Id, set => set
                .WithLength(5)
                .WithLeftPadding('0'))
            .WithMember(o => o.Description, set => set
                .WithLength(25)
                .WithRightPadding(' '))
            .WithMember(o => o.NullableInt, set => set
                .WithLength(5)
                .AllowNull("=Null")
                .WithLeftPadding('0'));

        return layout;
    }

    // you can also register a StringNormalizer function to convert input into the FixedLengthLineBuilder
    // to a string compatible with the specifications for your target File. 
    //
    // Note that the StringNormalizer function is only used when creating/building files. Not when reading/parsing files.
    //
    // example:
    public IFixedLayout<TestObject> GetLayout()
    {
        IFixedLayout<TestObject> layout = new FixedLayout<TestObject>()
            .WithMember(o => o.Description, set => set
                .WithLength(25)
                .WithRightPadding(' ')
                .WithStringNormalizer((input) => {
                     // the normalization to FormD splits accented letters in accents+letters,
                     // the rest aftet that removes those accents (and other non-spacing characters) from the ouput
                     // So unicode L'été becomes L'ete
                     return new string(
                        input.Normalize(System.Text.NormalizationForm.FormD)
                             .ToCharArray()
                             .Where(c => CharUnicodeInfo.GetUnicodeCategory(c) != UnicodeCategory.NonSpacingMark)
                             .ToArray());
                }))

        return layout;
    }

Attribute mapping

Delimited

using FlatFile.Delimited.Attributes;

[DelimitedFile(Delimiter = ";", Quotes = "\"")]
public class TestObject
{
    [DelimitedField(1)]
    public int Id { get; set; }

    [DelimitedField(2)]
    public string Description { get; set; }

    [DelimitedField(3, NullValue = "=Null")]
    public int? NullableInt { get; set; }
}

Fixed

using FlatFile.FixedLength;
using FlatFile.FixedLength.Attributes;

[FixedLengthFile]
public class TestObject
{
    [FixedLengthField(1, 5, PaddingChar = '0')]
    public int Id { get; set; }

    [FixedLengthField(2, 25, PaddingChar = ' ', Padding = Padding.Right)]
    public string Description { get; set; }

    [FixedLengthField(2, 5, PaddingChar = '0', NullValue = "=Null")]
    public int? NullableInt { get; set; }
}

Read from stream

With layout

var layout = new FixedSampleRecordLayout();
var factory = new FixedLengthFileEngineFactory();
using (var stream = new MemoryStream(Encoding.UTF8.GetBytes(FixedFileSample)))
{
    var flatFile = factory.GetEngine(layout);

    var records = flatFile.Read<FixedSampleRecord>(stream).ToArray();
}

With attribute-mapping

var factory = new FixedLengthFileEngineFactory();
using (var stream = new MemoryStream(Encoding.UTF8.GetBytes(FixedFileSample)))
{
    var flatFile = factory.GetEngine<FixedSampleRecord>();

    var records = flatFile.Read<FixedSampleRecord>(stream).ToArray();
}

With multiple fixed record types

var factory = new FixedLengthFileEngineFactory();
using (var stream = new MemoryStream(Encoding.UTF8.GetBytes(FixedFileSample)))
{
    // If using attribute mapping, pass an array of record types
    // rather than layout instances
    var layouts = new ILayoutDescriptor<IFixedFieldSettingsContainer>[]
    {
        new HeaderRecordLayout(),
        new DetailRecordLayout(),
        new TrailerRecordLayout()
    };
    var flatFile = factory.GetEngine(layouts,
        line =>
        {
            // For each line, return the proper record type.
            // The mapping for this line will be loaded based on that type.
            // In this simple example, the first character determines the
            // record type.
            if (String.IsNullOrEmpty(line) || line.Length < 1) return null;
            switch (line[0])
            {
                case 'H':
                    return typeof (HeaderRecord);
                case 'D':
                    return typeof (DetailRecord);
                case 'T':
                    return typeof (TrailerRecord);
            }
            return null;
        });

    flatFile.Read(stream);
    var header = flatFile.GetRecords<HeaderRecord>().FirstOrDefault();
    var records = flatFile.GetRecords<DetailRecord>();
    var trailer = flatFile.GetRecords<TrailerRecord>().FirstOrDefault();
}

Write to stream

With layout

var sampleRecords = GetRecords();
var layout = new FixedSampleRecordLayout();
var factory = new FixedLengthFileEngineFactory();
using (var stream = new MemoryStream())
{
    var flatFile = factory.GetEngine(layout);

    flatFile.Write<FixedSampleRecord>(stream, sampleRecords);
}

With attribute-mapping

var sampleRecords = GetRecords();
var factory = new FixedLengthFileEngineFactory();
using (var stream = new MemoryStream())
{
    var flatFile = factory.GetEngine<FixedSampleRecord>();

    flatFile.Write<FixedSampleRecord>(stream, sampleRecords);
}