refactor Preprocessor

Tests/ContentPipeline/EffectProcessorTests.cs

@@ -58,9 +58,9 @@ public void TestPreprocessor()
             var processorContext = new TestProcessorContext(TargetPlatform.Windows, Path.ChangeExtension(filename, ".xnb"));
 
             // Preprocess.
-            Preprocessor pp = new Preprocessor();
+            Preprocessor pp = new Preprocessor(effect, processorContext, fullFilePath);
             pp.AddMacro("TEST2", "1");
-            var mgPreprocessed = pp.Preprocess(effect, processorContext, fullFilePath);
+            var mgPreprocessed = pp.Preprocess();
 
             Assert.That(processorContext._dependencies, Has.Count.EqualTo(1));
             Assert.That(Path.GetFileName(processorContext._dependencies[0]), Is.EqualTo("PreprocessorInclude.fxh"));

src/Xna.Framework.Content.Pipeline.Graphics.MojoProcessor/EffectCompiler/Preprocessor.cs

@@ -2,7 +2,7 @@
 // This file is subject to the terms and conditions defined in
 // file 'LICENSE.txt', which is part of this source code package.
 
-// Copyright (C)2022 Nick Kastellanos
+// Copyright (C)2022-2024 Nick Kastellanos
 
 using System;
 using System.Collections.Generic;
@@ -13,189 +13,197 @@
 
 namespace Microsoft.Xna.Framework.Content.Pipeline.EffectCompiler
 {
-    public class Preprocessor
+    public class Preprocessor : PreprocessorListener, VirtualFileSystem
     {
-        private CppNet.Preprocessor _pp = new CppNet.Preprocessor();
+        private EffectContent _input;
+        private ContentProcessorContext _context;
+        private string _fullFilePath;
+        private CppNet.Preprocessor _pp;
 
+        public Preprocessor(EffectContent input, ContentProcessorContext context, string fullFilePath)
+        {
+            this._input = input;
+            this._context = context;
+            this._fullFilePath = fullFilePath;
+
+            _pp = new CppNet.Preprocessor();
+            _pp.EmitExtraLineInfo = false;
+            _pp.addFeature(Feature.LINEMARKERS);
+            _pp.setListener(this);
+            _pp.setFileSystem(this);
+        }
 
         internal void AddMacro(string name, string value)
         {
             _pp.addMacro(name, value);
         }
 
-        public string Preprocess(EffectContent input, ContentProcessorContext context, string fullFilePath)
+        internal void AddMacros(IEnumerable<KeyValuePair<string, string>> macros)
         {
-            List<string> dependencies = new List<string>();
-
-            _pp.EmitExtraLineInfo = false;
-            _pp.addFeature(Feature.LINEMARKERS);
-            _pp.setListener(new MGErrorListener(context.Logger));
-            _pp.setFileSystem(new MGFileSystem(dependencies));
-            _pp.setQuoteIncludePath(new List<string> { Path.GetDirectoryName(fullFilePath) });
-            
-            string effectCode = input.EffectCode;
+            foreach (KeyValuePair<string, string> macro in macros)
+                _pp.addMacro(macro.Key, macro.Value);
+        }
+
+        public string Preprocess()
+        {
+            _pp.setQuoteIncludePath(new List<string> { Path.GetDirectoryName(_fullFilePath) });
+
+            string effectCode = _input.EffectCode;
             effectCode = effectCode.Replace("#line", "//--WORKAROUND#line");
 
-            _pp.addInput(new MGStringLexerSource(effectCode, true, fullFilePath));
+            StringLexerSource inputSource = new PPStringLexerSource(effectCode, true, _fullFilePath);
+            _pp.addInput(inputSource);
 
             StringBuilder result = new StringBuilder();
 
-            bool endOfStream = false;
-            while (!endOfStream)
+            while (true)
             {
                 Token token = _pp.token();
-                switch (token.getType())
+                int tokenType = token.getType();
+
+                switch (tokenType)
                 {
                     case CppNet.Token.EOF:
-                        endOfStream = true;
-                        break;
+                        return result.ToString();
+
                     case CppNet.Token.CPPCOMMENT:
-                        if (token.getText().StartsWith("//--WORKAROUND#line"))
                         {
-                            result.Append(token.getText().Replace("//--WORKAROUND#line", "#line"));
+                            string tokenText = token.getText();
+                            if (tokenText.StartsWith("//--WORKAROUND#line"))
+                            {
+                                result.Append(tokenText.Replace("//--WORKAROUND#line", "#line"));
+                            }
                         }
                         break;
+
                     case CppNet.Token.CCOMMENT:
-                    {
-                        string tokenText = token.getText();
-                        if (tokenText != null)
                         {
-                            // Need to preserve line breaks so that line numbers are correct.
-                            foreach (char c in tokenText)
-                                if (c == '\n')
-                                    result.Append(c);
+                            string tokenText = token.getText();
+                            if (tokenText != null)
+                            {
+                                // Need to preserve line breaks so that line numbers are correct.
+                                foreach (char c in tokenText)
+                                    if (c == '\n')
+                                        result.Append(c);
+                            }
                         }
                         break;
-                    }
+
                     default:
-                    {
+                        {
                             string tokenText = token.getText();
-                        if (tokenText != null)
-                            result.Append(tokenText);
+                            if (tokenText != null)
+                                result.Append(tokenText);
+                        }
                         break;
-                    }
                 }
             }
 
-            // Add the include dependencies so that if they change
-            // it will trigger a rebuild of this effect.
-            foreach (string dep in dependencies)
-                context.AddDependency(dep);
+        }
+
 
-            return result.ToString();
+        #region PreprocessorListener
+
+        void PreprocessorListener.handleWarning(Source source, int line, int column, string msg)
+        {
+            string file = ((PPStringLexerSource)source).Path;
+            ContentIdentity contentIdentity = new ContentIdentity(file, null, line + "," + column);
+            _context.Logger.LogWarning(null, contentIdentity, msg);
         }
 
-        private class MGFileSystem : VirtualFileSystem
+        void PreprocessorListener.handleError(Source source, int line, int column, string msg)
         {
-            private readonly List<string> _dependencies;
+            string file = ((PPStringLexerSource)source).Path;
+            ContentIdentity contentIdentity = new ContentIdentity(file, null, line + "," + column);
+            throw new InvalidContentException(msg, contentIdentity);
+        }
 
-            public MGFileSystem(List<string> dependencies)
-            {
-                _dependencies = dependencies;
-            }
+        void PreprocessorListener.handleSourceChange(Source source, string ev)
+        {
 
-            public VirtualFile getFile(string path)
-            {
-                return new MGFile(path, _dependencies);
-            }
+        }
 
-            public VirtualFile getFile(string dir, string name)
-            {
-                return new MGFile(Path.Combine(dir, name), _dependencies);
-            }
+        #endregion PreprocessorListener
+
+
+        #region VirtualFileSystem
+
+        VirtualFile VirtualFileSystem.getFile(string path)
+        {
+            return new PPVirtualFile((VirtualFileSystem)this, path);
+        }
+
+        VirtualFile VirtualFileSystem.getFile(string dir, string name)
+        {
+            return new PPVirtualFile((VirtualFileSystem)this, Path.Combine(dir, name));
         }
 
-        private class MGFile : VirtualFile
+        #endregion VirtualFileSystem
+
+
+        private class PPVirtualFile : VirtualFile
         {
-            private readonly List<string> _dependencies;
+            VirtualFileSystem _virtualFileSystem;
             private readonly string _path;
 
-            public MGFile(string path, List<string> dependencies)
+            public PPVirtualFile(VirtualFileSystem virtualFileSystem, string path)
             {
-                _dependencies = dependencies;
-                _path = Path.GetFullPath(path);
+                this._virtualFileSystem = virtualFileSystem;
+                this._path = Path.GetFullPath(path);
             }
 
-            public bool isFile()
+            bool VirtualFile.isFile()
             {
                 return File.Exists(_path) && !File.GetAttributes(_path).HasFlag(FileAttributes.Directory);
             }
 
-            public string getPath()
+            string VirtualFile.getPath()
             {
                 return _path;
             }
 
-            public string getName()
+            string VirtualFile.getName()
             {
                 return Path.GetFileName(_path);
             }
 
-            public VirtualFile getParentFile()
+            VirtualFile VirtualFile.getParentFile()
             {
-                return new MGFile(Path.GetDirectoryName(_path), _dependencies);
+                return new PPVirtualFile(_virtualFileSystem, Path.GetDirectoryName(_path));
             }
 
-            public VirtualFile getChildFile(string name)
+            VirtualFile VirtualFile.getChildFile(string name)
             {
-                return new MGFile(Path.Combine(_path, name), _dependencies);
+                return new PPVirtualFile(_virtualFileSystem, Path.Combine(_path, name));
             }
 
-            public Source getSource()
+            Source VirtualFile.getSource()
             {
-                if (!_dependencies.Contains(_path))
-                    _dependencies.Add(_path);
-                return new MGStringLexerSource(AppendNewlineIfNonePresent(File.ReadAllText(_path)), true, _path);
-            }
+                // Add the include dependencies so that if they change
+                // it will trigger a rebuild of this effect.
+                ((Preprocessor)_virtualFileSystem)._context.AddDependency(_path);
 
-            private static string AppendNewlineIfNonePresent(string text)
-            {
-                if (!text.EndsWith("\n"))
-                    return text + "\n";
-                return text;
+                string effectCode = File.ReadAllText(_path);
+                // Append new line if none present.
+                if (!effectCode.EndsWith("\n"))
+                    effectCode += "\n";
+
+                StringLexerSource source = new PPStringLexerSource(effectCode, true, _path);
+                return source;
             }
+
         }
 
-        private class MGStringLexerSource : StringLexerSource
+        private class PPStringLexerSource : StringLexerSource
         {
             public string Path { get; private set; }
 
-            public MGStringLexerSource(string str, bool ppvalid, string fileName)
+            public PPStringLexerSource(string str, bool ppvalid, string fileName)
                 : base(str.Replace("\r\n", "\n"), ppvalid, fileName)
             {
                 Path = fileName;
             }
         }
 
-        private class MGErrorListener : PreprocessorListener
-        {
-            private readonly ContentBuildLogger _logger;
-
-            public MGErrorListener(ContentBuildLogger logger)
-            {
-                _logger = logger;
-            }
-
-            public void handleWarning(Source source, int line, int column, string msg)
-            {
-                _logger.LogWarning(null, CreateContentIdentity(source, line, column), msg);
-            }
-
-            public void handleError(Source source, int line, int column, string msg)
-            {
-                throw new InvalidContentException(msg, CreateContentIdentity(source, line, column));
-            }
-
-            private static ContentIdentity CreateContentIdentity(Source source, int line, int column)
-            {
-                string file = ((MGStringLexerSource)source).Path;
-                return new ContentIdentity(file, null, line + "," + column);
-            }
-
-            public void handleSourceChange(Source source, string ev)
-            {
-
-            }
-        }
     }
 }

src/Xna.Framework.Content.Pipeline.Graphics.MojoProcessor/Processors/EffectProcessor.cs

@@ -150,7 +150,7 @@ public static ShaderProfile FromType(ShaderProfileType profileType)
         // resolving all #includes and macros.
         private string Preprocess(EffectContent input, ContentProcessorContext context, ShaderProfile shaderProfile, string fullFilePath)
         {
-            Preprocessor pp = new Preprocessor();
+            Preprocessor pp = new Preprocessor(input, context, fullFilePath);
 
             pp.AddMacro("__KNIFX__", "1");
 
@@ -166,8 +166,7 @@ private string Preprocess(EffectContent input, ContentProcessorContext context,
                 pp.AddMacro("DEBUG", "1");
             }
 
-            foreach (KeyValuePair<string,string> macro in shaderProfile.GetMacros())
-                pp.AddMacro(macro.Key, macro.Value);
+            pp.AddMacros(shaderProfile.GetMacros());
 
             if (!string.IsNullOrEmpty(Defines))
             {
@@ -191,7 +190,7 @@ private string Preprocess(EffectContent input, ContentProcessorContext context,
                 }
             }
 
-            string effectCode = pp.Preprocess(input, context, fullFilePath);
+            string effectCode = pp.Preprocess();
 
             return effectCode;
         }