From 8f6975b5191479c074deb291e6755950bd1fe0d5 Mon Sep 17 00:00:00 2001 From: Curly <36010519+neon-JS@users.noreply.github.com> Date: Fri, 30 Aug 2024 16:33:56 +0200 Subject: [PATCH 1/3] chore: Added tools project for internal tasks - new cs-project for internal tasks - added a new task for updating our list of file types in our readme easily. --- MagicBytesValidator.Tools/Imports.cs | 3 + .../MagicBytesValidator.Tools.csproj | 12 ++++ MagicBytesValidator.Tools/Program.cs | 56 +++++++++++++++++++ MagicBytesValidator.Tools/README.md | 20 +++++++ MagicBytesValidator.sln | 7 +++ 5 files changed, 98 insertions(+) create mode 100644 MagicBytesValidator.Tools/Imports.cs create mode 100644 MagicBytesValidator.Tools/MagicBytesValidator.Tools.csproj create mode 100644 MagicBytesValidator.Tools/Program.cs create mode 100644 MagicBytesValidator.Tools/README.md diff --git a/MagicBytesValidator.Tools/Imports.cs b/MagicBytesValidator.Tools/Imports.cs new file mode 100644 index 0000000..64399bb --- /dev/null +++ b/MagicBytesValidator.Tools/Imports.cs @@ -0,0 +1,3 @@ +// Global using directives + +global using MagicBytesValidator.Services; diff --git a/MagicBytesValidator.Tools/MagicBytesValidator.Tools.csproj b/MagicBytesValidator.Tools/MagicBytesValidator.Tools.csproj new file mode 100644 index 0000000..3f345e0 --- /dev/null +++ b/MagicBytesValidator.Tools/MagicBytesValidator.Tools.csproj @@ -0,0 +1,12 @@ + + + + enable + exe + + + + + + + diff --git a/MagicBytesValidator.Tools/Program.cs b/MagicBytesValidator.Tools/Program.cs new file mode 100644 index 0000000..17cf441 --- /dev/null +++ b/MagicBytesValidator.Tools/Program.cs @@ -0,0 +1,56 @@ +namespace MagicBytesValidator.Tools; + +public class Program +{ + public static void Main(string[] args) + { + var argument = args.FirstOrDefault()?.ToLower() ?? "help"; + + switch (argument) + { + case "table": + PrintTypesTable(); + break; + case "help": + PrintHelp(); + break; + default: + Console.Error.WriteLine($"Unknown option '{argument}'. Maybe try using 'help'?"); + break; + } + } + + private static void PrintTypesTable() + { + Console.WriteLine(); + Console.WriteLine("|FileType|Extensions|MIME Types|"); + Console.WriteLine("|-|-|-|"); + + foreach (var fileType in (new Mapping()).FileTypes) + { + var mimeTypeList = string.Join(", ", fileType.MimeTypes); + var extensionList = string.Join(", ", fileType.Extensions); + + Console.WriteLine($"|{fileType.GetType().Name.ToUpper()}|{extensionList}|{mimeTypeList}|"); + } + } + + private static void PrintHelp() + { + Console.WriteLine(""" + + Usage: dotnet run -- [tool] + + Available tools: + + - table + Prints a markdown table containing all known file types, including extensions and MIME types. + Useful for updating the project README file. + + - help + Shows this help + + (traperto GmbH, 2024) + """); + } +} \ No newline at end of file diff --git a/MagicBytesValidator.Tools/README.md b/MagicBytesValidator.Tools/README.md new file mode 100644 index 0000000..f4200c6 --- /dev/null +++ b/MagicBytesValidator.Tools/README.md @@ -0,0 +1,20 @@ +# MagicByteValidator.Tools + +## About this project + +This is a helper project for internal tasks, e. g. updating the main [README.md](../README.md). +**It is not meant to be used by non-traperto developers.** +If you want to debug your file types or sample files, use the [MagicBytesValidator.CLI](../MagicBytesValidator.CLI/) +project. + +## Usage + +```shell +dotnet run --project MagicBytesValidator.Tools -- [tool] +``` + +### Available tools + +- _table_: Prints a markdown table containing all known file types, including extensions and MIME types. + Useful for updating the project README file. +- _help_: Shows help page of this project. \ No newline at end of file diff --git a/MagicBytesValidator.sln b/MagicBytesValidator.sln index c0b9902..a75d130 100644 --- a/MagicBytesValidator.sln +++ b/MagicBytesValidator.sln @@ -1,11 +1,14 @@  Microsoft Visual Studio Solution File, Format Version 12.00 +# Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "MagicBytesValidator", "MagicBytesValidator\MagicBytesValidator.csproj", "{E944E89C-D807-45B8-9173-33DD42116BB3}" EndProject Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "MagicBytesValidator.Tests", "MagicBytesValidator.Tests\MagicBytesValidator.Tests.csproj", "{B23AB832-C861-448A-A44F-DC46E1884FB0}" EndProject Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "MagicBytesValidator.CLI", "MagicBytesValidator.CLI\MagicBytesValidator.CLI.csproj", "{1B36073D-9033-4E57-82F8-37810880450D}" EndProject +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "MagicBytesValidator.Tools", "MagicBytesValidator.Tools\MagicBytesValidator.Tools.csproj", "{01F437D7-D727-4975-8D04-469AE89B76AD}" +EndProject Global GlobalSection(SolutionConfigurationPlatforms) = preSolution Debug|Any CPU = Debug|Any CPU @@ -24,5 +27,9 @@ Global {1B36073D-9033-4E57-82F8-37810880450D}.Debug|Any CPU.Build.0 = Debug|Any CPU {1B36073D-9033-4E57-82F8-37810880450D}.Release|Any CPU.ActiveCfg = Release|Any CPU {1B36073D-9033-4E57-82F8-37810880450D}.Release|Any CPU.Build.0 = Release|Any CPU + {01F437D7-D727-4975-8D04-469AE89B76AD}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {01F437D7-D727-4975-8D04-469AE89B76AD}.Debug|Any CPU.Build.0 = Debug|Any CPU + {01F437D7-D727-4975-8D04-469AE89B76AD}.Release|Any CPU.ActiveCfg = Release|Any CPU + {01F437D7-D727-4975-8D04-469AE89B76AD}.Release|Any CPU.Build.0 = Release|Any CPU EndGlobalSection EndGlobal From c4740f4c3b2fd050ea61db83cd10b8bfccebbaa6 Mon Sep 17 00:00:00 2001 From: Curly <36010519+neon-JS@users.noreply.github.com> Date: Fri, 30 Aug 2024 16:35:09 +0200 Subject: [PATCH 2/3] docs: Updated README - fixed typos and streamlined texting - updated the list of known file types --- README.md | 122 +++++++++++++++++++++++++++++------------------------- 1 file changed, 66 insertions(+), 56 deletions(-) diff --git a/README.md b/README.md index 5c38b85..7046968 100644 --- a/README.md +++ b/README.md @@ -1,6 +1,6 @@ # MagicBytesValidator -Recognize filetypes from `Streams` or `IFormfiles` using mime types or file extensions and validate them against the magic bytes according to the filetypes. -The existing `FileTypes` can be expanded in various ways. +Recognize file types from `Stream`s or `IFormFile`s using MIME types or file extensions and validate them against the magic bytes according to the file types. +The existing `IFileType`s can be expanded in various ways. ### How to install? @@ -21,19 +21,19 @@ dotnet add package MagicBytesValidator --version 2.0.2 ### How to use it? -- Create new instances of the validator & providers: +- Create new instances of the validator and providers: ```c# var validator = new MagicBytesValidator.Services.Validator(); var formFileTypeProvider = new MagicBytesValidator.Services.Http.FormFileTypeProvider(); var streamFileTypeProvider = new MagicBytesValidator.Services.Streams.StreamFileTypeProvider(); ``` -- Find a filetype by extension or mimetype: +- Find a file type by extension or MIME type: ```c# var pngFileType = validator.Mapping.FindByExtension("png"); var pdfFileType = validator.Mapping.FindByMimeType("application/pdf"); ``` -- Determine & validate a filetype by uploaded IFormFile: +- Determine and validate a file type by uploaded IFormFile: ```c# var fileType = await formFileTypeProvider.FindValidatedTypeAsync(formFile, null, CancellationToken.None); ``` @@ -43,12 +43,12 @@ var fileType = await formFileTypeProvider.FindValidatedTypeAsync(formFile, null, var fileType = await streamFileTypeProvider.TryFindUnambiguousAsync(fileStream, CancellationToken.None); ``` -- Check a file with its stream and filetype: +- Check a file with its stream and file type: ```c# var isValid = await validator.IsValidAsync(memoryStream, fileType, CancellationToken.None); ``` -#### Expand the filetype mapping +#### Expand the file type mapping - Get mapping: ```c# @@ -62,29 +62,29 @@ var mapping = formFileTypeProvider.Mapping; var mapping = new MagicBytesValidator.Services.Mapping(); ``` -- Register a single Filetype: +- Register a single `FileByteFilter`: ```c# mapping.Register( new FileByteFilter( - "traperto/trp", // mime type + "traperto/trp", // MIME type new[] { "trp" } // file extensions ) { // magic byte sequences StartsWith([ 0x78, 0x6c, 0x2f, 0x5f, 0x72, 0x65 ]) - .EndsWith([ + .EndsWith([ 0xFF, 0xFF ]) } ) ``` -- FileTypes with specific offset checks: +- `FileByteFilter`s with specific offset checks: ```c# mapping.Register( new FileByteFilter( - "traperto/trp", // mime type + "traperto/trp", // MIME type new[] { "trp" } // file extensions ) { // magic byte sequences @@ -92,22 +92,21 @@ mapping.Register( } ) ``` -ByteCheck allows for negative offset values to look for a specific offset counting from the end of file +`ByteCheck` allows for negative offset values to look for a specific offset counting from the end of file. - Register a list of filetypes: ```c# mapping.Register(listOfFileTypes); ``` -You can also create variants of `IFileType` and register them by passing the Assembly of the new FileTypes, e.g. -`mapping.Register(typeof(CustomFileType).Assembly);`. This will register all FileTypes of the given Assembly that are also -not abstract and have an empty constructor! +You can also create variants of `IFileType` and register them by passing the Assembly of the new `IFileType`s, e.g. +`mapping.Register(typeof(CustomFileType).Assembly);`. This will register all `IFileType`s of the given assembly that are also not abstract and have an empty constructor. ```c# public class CustomFileType : FileTypeWithStartSequences { public CustomFileType() : base( - "traperto/trp", // mime type + "traperto/trp", // MIME type new[] { "trp" }, // file extensions new[] { // magic byte sequences new byte[] { 0x74, 0x72, 0x61, 0x70, 0x65, 0x72, 0x74, 0x6f } @@ -123,54 +122,65 @@ _mapping.Register(assembly); ### CLI There's a CLI tool (_MagicBytesValidator.CLI_) which can be used to determine -MIME types for a local file by calling the following command. +MIME types for a local file by calling the following command: + ```shell dotnet run --project MagicBytesValidator.CLI -- [PATH] ``` This can be useful when debugging or validating newly added FileTypes. -### List of Filetypes - -| Mimetype | Extension | Magicbytes (decimal) | -|-------------------------------------------------|--------------------------------------------|------------------------------------------------------------------------------------------------------------------------------------| -| audio/x-pn-realaudio-plugin | rpm | 237 171 238 219 | -| application/octet-stream | bin
file
com
class
ini | | -| video/3gpp | 3gp | 102 116 121 112 51 103 | -| image/x-icon | ico | 0 0 1 0 | -| image/gif | gif | | -| image/tiff | tif
tiff | | -| image/jpeg | jpg
jpeg
jpe | | -| image/png | png | 137 80 78 71 13 10 26 10 | -| video/ogg | ogg
ogv | 79 103 103 83 | -| audio/basic | snd
au | | -| application/dsptype | tsp | 77 90 | -| text/plain | txt | | -| application/zip | zip | 80 75 3 4 | -| application | docx
xlsx | 80 75 7 8 | -| application/vnd.oasis.opendocument.presentation | odp | 80 75 7 8 | -| application/vnd.oasis.opendocument.spreadsheet | ods | 80 75 7 8 | -| application/vnd.oasis.opendocument.text | odt | 80 75 7 8 | -| audio/mpeg | mp3 | 73 68 51 | -| image/bmp | bmp | 66 77 | -| audio/x-midi | midi
mid | 77 84 104 100 | -| application/msword | doc
dot | 208 207 17 224 161 177 26 255 | -| application/msexcel | xlx
xla | 208 207 17 224 161 177 26 255 | -| application/mspowerpoint | ppt
ppz
pps
pt | 208 207 17 224 161 177 26 225 | -| application/gzip | gz | 31 139 | -| video/webm | webm | 26 69 223 163 | -| application/rtf | rtf | 123 92 114 116 102 49 | -| text/tab-separated-values | tsv | 71 | -| video/mpeg | mpg
mpeg
mpe | | -| video/mp4 | mp4 | | -| image/x-portable-bitmap | pbm | 80 49 10 | -| image/x-portable-graymap | pgm | 80 50 10 | -| image/x-portable-pixmap | ppm | 80 51 10 | -| application/pdf | pdf | _multiple_ | +### List of file types + +|FileType|Extensions|MIME Types| +|-|-|-| +|AIF|aif, aiff, aifc|audio/x-aiff| +|BIN|bin, file, com, class, ini|application/octet-stream| +|BMP|bmp|image/bmp| +|CAB|cab|application/vnd.ms-cab-compressed, application/x-cab-compressed| +|DOC|doc, dot|application/msword| +|DOCX|docx|application/vnd.openxmlformats-officedocument.wordprocessingml.document| +|DXR|dxr, dcr, dir|application/x-director| +|EXE|exe, com, dll, drv, pif, qts, qtx , sys, acm, ax, cpl, fon, ocx, olb, scr, vbx, vxd, mui, iec, ime, rs, tsp, efi|application/x-dosexec, application/x-msdos-program| +|GIF|gif|image/gif| +|GZ|gz|application/gzip| +|ICO|ico|image/x-icon| +|JPG|jpg, jpeg, jpe, jif, jfif, jfi|image/jpeg| +|MIDI|midi, mid|audio/x-midi| +|MP3|mp3|audio/mpeg| +|MP4|mp4|video/mp4| +|MPG|mpg, mpeg, mpe, m2p, vob|video/mpeg| +|ODP|odp|application/vnd.oasis.opendocument.presentation| +|ODS|ods|application/vnd.oasis.opendocument.spreadsheet| +|ODT|odt|application/vnd.oasis.opendocument.text| +|OGV|ogv, ogg, oga|video/ogg| +|PBM|pbm|image/x-portable-bitmap| +|PDF|pdf|application/pdf| +|PGM|pgm|image/x-portable-graymap| +|PNG|png|image/png| +|PPM|ppm|image/x-portable-pixmap| +|PPT|ppt, ppz, pps, pot|application/mspowerpoint, application/vnd.ms-powerpoint| +|PPTX|pptx|application/vnd.openxmlformats-officedocument.presentationml.presentation| +|RAR|rar|application/vnd.rar, application/x-rar-compressed| +|RPM|rpm|application/x-rpm, application/x-redhat-package-manager| +|RTF|rtf|application/rtf| +|SND|snd, au|audio/basic| +|SVG|svg, svgz|image/svg+xml| +|SWF|swf|application/x-shockwave-flash| +|3GP|3gp|video/3gpp| +|TIF|tif, tiff|image/tiff| +|TSV|ts, tsv, tsa, mpg, mpeg|video/mp2t| +|TXT|txt|text/plain| +|WEBM|mkv, mka, mks, mk3d, webm|video/webm| +|XLS|xls, xla|application/msexcel| +|XLSX|xlsx|application/vnd.openxmlformats-officedocument.spreadsheetml.sheet| +|XML|xml|application/xml, text/xml| +|Z|z|application/x-compress| +|ZIP|zip|application/zip, application/x-zip-compressed| ### What is the licence? -MIT License +[MIT License](./LICENSE) ``` ▓▓ ▓▓▓▓▓▓▓▓▓▓ From ff30292f9d496c9443e3e9acd5e530b542aac282 Mon Sep 17 00:00:00 2001 From: Curly <36010519+neon-JS@users.noreply.github.com> Date: Fri, 30 Aug 2024 16:48:14 +0200 Subject: [PATCH 3/3] test: Added test runner as `dotnet test` won't work otherwise --- MagicBytesValidator.Tests/MagicBytesValidator.Tests.csproj | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/MagicBytesValidator.Tests/MagicBytesValidator.Tests.csproj b/MagicBytesValidator.Tests/MagicBytesValidator.Tests.csproj index f7dd2bd..402862b 100644 --- a/MagicBytesValidator.Tests/MagicBytesValidator.Tests.csproj +++ b/MagicBytesValidator.Tests/MagicBytesValidator.Tests.csproj @@ -8,6 +8,10 @@ + + all + runtime; build; native; contentfiles; analyzers + runtime; build; native; contentfiles; analyzers; buildtransitive all