Updates to Parsing

benchmark.md

@@ -1,9 +1,9 @@
 # System informations 
- {"gpu":"","cpuName":"Apple M1","freeRam":152.375,"totalRam":8192}
+ {"gpu":"","cpuName":"Apple M1","freeRam":18.25390625,"totalRam":8192}
  _________ 
 | filename | Size (mo) | Time to open model (ms) | Time to execute all (ms) | Total ifc entities | Total meshes | Total geometries | total errors |
 |-------|-------|-------|-------|-------|-------|-------|-------|
-| ifcfiles/advanced_model.ifc.test | 33.67 | 413 | 1623 | 594374 | 6401 | 14120 | 0 |
- ifcfiles/dental_clinic.ifc.test | 12.4 | 140 | 376 | 209259 | 2586 | 2626 | 0 |
- ifcfiles/example.ifc.test | 0.39 | 5 | 9 | 6487 | 115 | 119 | 0 |
- ifcfiles/tested_sample_project.ifc.test | 0.68 | 7 | 97 | 14119 | 93 | 98 | 0 |
+| ifcfiles/advanced_model.ifc.test | 33.67 | 428 | 2711 | 594374 | 6401 | 14120 | 0 |
+ ifcfiles/dental_clinic.ifc.test | 12.4 | 243 | 1149 | 209259 | 2586 | 2626 | 176 |
+ ifcfiles/example.ifc.test | 0.39 | 6 | 12 | 6487 | 115 | 119 | 0 |
+ ifcfiles/tested_sample_project.ifc.test | 0.68 | 9 | 135 | 14119 | 93 | 98 | 0 |

src/wasm/CMakeLists.txt

@@ -76,7 +76,7 @@ if (EMSCRIPTEN)
 	target_compile_options(web-ifc PUBLIC "-Wextra")
 	target_compile_options(web-ifc PUBLIC "-Wpedantic")
 	target_compile_options(web-ifc PUBLIC "-pedantic")
-	if (release)
+	if (RELEASE)
 		target_compile_options(web-ifc PUBLIC "-O3")
 	endif()
 	set_target_properties(web-ifc PROPERTIES LINK_FLAGS "${DEBUG_FLAG} --bind -flto --define-macro=REAL_T_IS_DOUBLE -s ALLOW_MEMORY_GROWTH=1 -s MAXIMUM_MEMORY=4GB -sSTACK_SIZE=5MB -s EXPORT_NAME=WebIFCWasm -s MODULARIZE=1 -s ENVIRONMENT=web")
@@ -92,7 +92,7 @@ if (EMSCRIPTEN)
 	target_compile_options(web-ifc-node PUBLIC "-Wextra")
 	target_compile_options(web-ifc-node PUBLIC "-Wpedantic")
 	target_compile_options(web-ifc-node PUBLIC "-pedantic")
-	if (release)
+	if (RELEASE)
 		target_compile_options(web-ifc-node PUBLIC "-O3")
 	endif()
 	set_target_properties(web-ifc-node PROPERTIES LINK_FLAGS "${DEBUG_FLAG} --bind -flto --define-macro=REAL_T_IS_DOUBLE -s ALLOW_MEMORY_GROWTH=1 -s MAXIMUM_MEMORY=4GB -sSTACK_SIZE=5MB -s EXPORT_NAME=WebIFCWasm -s MODULARIZE=1 ")
@@ -111,7 +111,7 @@ if (EMSCRIPTEN)
 	target_compile_options(web-ifc-mt PUBLIC "-Wextra")
 	target_compile_options(web-ifc-mt PUBLIC "-Wpedantic")
 	target_compile_options(web-ifc-mt PUBLIC "-pedantic")
-	if (release)
+	if (RELEASE)
 		target_compile_options(web-ifc-mt PUBLIC "-O3")
 	endif()
 	set_target_properties(web-ifc-mt PROPERTIES LINK_FLAGS "${DEBUG_FLAG} -pthread -s PTHREAD_POOL_SIZE=navigator.hardwareConcurrency --bind -flto --define-macro=REAL_T_IS_DOUBLE -sSTACK_SIZE=5MB -s ALLOW_MEMORY_GROWTH=1 -s MAXIMUM_MEMORY=4GB -s EXPORT_NAME=WebIFCWasm -s MODULARIZE=1 -s ENVIRONMENT=web,worker")
@@ -133,7 +133,7 @@ if (NOT EMSCRIPTEN)
 	target_compile_options(web-ifc-test PUBLIC "-Wextra")
 	target_compile_options(web-ifc-test PUBLIC "-Wpedantic")
 	target_compile_options(web-ifc-test PUBLIC "-pedantic")
-	if (release)
+	if (RELEASE)
 		target_compile_options(web-ifc-test PUBLIC "-O3")
 	endif()
 	add_test (web-ifc-test web-ifc-test)
@@ -155,7 +155,8 @@ if (NOT EMSCRIPTEN)
 	#comment these to prevent debug files being generated
 	target_compile_options(web-ifc PUBLIC "-DCSG_DEBUG_OUTPUT")
 	target_compile_options(web-ifc PUBLIC "-DDEBUG_DUMP_SVG")
-	if (release)
+	if (RELEASE)
 		target_compile_options(web-ifc PUBLIC "-O3")
+		set_target_properties(web-ifc PROPERTIES LINK_FLAGS "-O3")
 	endif()
 endif()

src/wasm/parsing/IfcLoader.cpp

@@ -18,9 +18,9 @@ namespace webifc::parsing {
    std::string p21encode(std::string_view input);
 
 
-   IfcLoader::IfcLoader(size_t tapeSize, uint8_t noChunks,utility::LoaderErrorHandler &errorHandler,schema::IfcSchemaManager &schemaManager) :_schemaManager(schemaManager), _errorHandler(errorHandler)
+   IfcLoader::IfcLoader(size_t tapeSize, uint32_t memoryLimit,utility::LoaderErrorHandler &errorHandler,schema::IfcSchemaManager &schemaManager) :_schemaManager(schemaManager), _errorHandler(errorHandler)
    { 
-     _tokenStream = new IfcTokenStream(tapeSize,noChunks);
+     _tokenStream = new IfcTokenStream(tapeSize,memoryLimit/tapeSize);
      _nullLine = new IfcLine();
      _nullLine->ifcType=0;
      _nullLine->tapeOffset=0;
@@ -34,13 +34,11 @@ namespace webifc::parsing {
 
    const std::vector<IfcHeaderLine> IfcLoader::GetHeaderLinesWithType(const uint32_t type) const
    { 
-     if (_ifcTypeToHeaderLineID.count(type)==0) return {};
-     auto &list = _ifcTypeToHeaderLineID.at(type);
-     std::vector<IfcHeaderLine> ret(list.size());
-
-     std::transform(list.begin(), list.end(), ret.begin(), [&](uint32_t lineID)
-              { return _headerLines[lineID]; });
-
+     std::vector<IfcHeaderLine> ret;
+     for (auto &line: _headerLines)
+     {
+        if (line.ifcType==type) ret.push_back(line);
+     }
      return ret;
    }
 
@@ -69,7 +67,6 @@ namespace webifc::parsing {
           }
       }
       return IFC2X3;
-
    }
 
    void IfcLoader::LoadFile(std::istream &requestData)
@@ -321,7 +318,6 @@ namespace webifc::parsing {
   							l.ifcType = currentIfcType;
   							l.lineIndex = static_cast<uint32_t>(_headerLines.size());
   							l.tapeOffset = currentTapeOffset;
-  							_ifcTypeToHeaderLineID[l.ifcType].push_back(l.lineIndex);
   							_headerLines.push_back(std::move(l));
   						}
   					}
@@ -383,13 +379,13 @@ namespace webifc::parsing {
 
    bool IfcLoader::IsValidExpressID(const uint32_t expressID) const
    {  
-   	 if (expressID ==0 || _lines[expressID-1]==_nullLine) return false;
+   	 if (expressID == 0 || expressID > _lines.size() || _lines[expressID-1]==_nullLine) return false;
      else return true;
    }
 
    const IfcLine &IfcLoader::GetLine(const uint32_t expressID) const
    { 
-      if (_lines[expressID-1]==_nullLine || expressID == 0) {
+      if (expressID == 0 || expressID > _lines.size()) {
         _errorHandler.ReportError(utility::LoaderErrorType::PARSING, "Attempt to Access Invalid ExpressID", expressID);
         return *_nullLine;
       }
@@ -475,7 +471,7 @@ namespace webifc::parsing {
 
   void IfcLoader::UpdateLineTape(const uint32_t expressID, const uint32_t type, const uint32_t start)
   {
-  	if (_lines.size() <expressID) _lines.resize(expressID,_nullLine);
+  	if (_lines.size() < expressID) _lines.resize(expressID,_nullLine);
     // new line?
     if (_lines[expressID-1] == _nullLine)
   	{
@@ -498,7 +494,6 @@ namespace webifc::parsing {
       l.ifcType = type;
       l.lineIndex = static_cast<uint32_t>(_headerLines.size());
       l.tapeOffset = start;
-      _ifcTypeToHeaderLineID[l.ifcType].push_back(l.lineIndex);
       _headerLines.push_back(std::move(l));
   }
 

src/wasm/parsing/IfcLoader.h

@@ -32,7 +32,7 @@ namespace webifc::parsing
 	class IfcLoader {
 
     public:
-      IfcLoader(size_t tapeSize, uint8_t noChunks,utility::LoaderErrorHandler &errorHandler,schema::IfcSchemaManager &schemaManager);  
+      IfcLoader(size_t tapeSize, uint32_t memoryLimit,utility::LoaderErrorHandler &errorHandler,schema::IfcSchemaManager &schemaManager);  
       ~IfcLoader();
       const std::vector<IfcHeaderLine> GetHeaderLinesWithType(const uint32_t type) const;
       void LoadFile(const std::function<uint32_t(char *, size_t, size_t)> &requestData);
@@ -82,7 +82,6 @@ namespace webifc::parsing
       std::vector<IfcLine*> _lines;
       std::vector<IfcHeaderLine> _headerLines;
       std::unordered_map<uint32_t, std::vector<uint32_t>> _ifcTypeToExpressID;
-      std::unordered_map<uint32_t, std::vector<uint32_t>> _ifcTypeToHeaderLineID;
       void ParseLines();
       void ArgumentOffset(const uint32_t argumentIndex) const;      
 	};

src/wasm/utility/LoaderSettings.h

@@ -13,6 +13,6 @@ namespace webifc::utility
 		bool COORDINATE_TO_ORIGIN = false;
 		uint16_t CIRCLE_SEGMENTS = 12;
 		uint32_t TAPE_SIZE = 67108864 ; // probably no need for anyone other than web-ifc devs to change this
-		uint8_t NO_CHUNKS =  25;
+		uint8_t MEMORY_LIMIT = 3221225472;
 	};
 }

src/wasm/web-ifc-api.cpp

@@ -25,7 +25,7 @@ struct ModelInfo
         ModelInfo(webifc::utility::LoaderSettings _settings, webifc::schema::IfcSchemaManager &_schemaManager) : schemaManager(_schemaManager), settings(_settings)
         {
             errorHandler = new webifc::utility::LoaderErrorHandler();
-            loader = new webifc::parsing::IfcLoader(_settings.TAPE_SIZE,_settings.NO_CHUNKS,*errorHandler,schemaManager);
+            loader = new webifc::parsing::IfcLoader(_settings.TAPE_SIZE,_settings.MEMORY_LIMIT,*errorHandler,schemaManager);
         }
 
         webifc::geometry::IfcGeometryProcessor * GetGeometryLoader()
@@ -447,7 +447,6 @@ bool ValidateExpressID(uint32_t modelID, uint32_t expressId)
     {
         return {};
     }
-
     return loader->IsValidExpressID(expressId);
 }
 
@@ -839,7 +838,7 @@ uint32_t GetLineType(uint32_t modelID, uint32_t expressID)
     {
         return 0;
     }
-    if (loader->GetMaxExpressId() < expressID || !loader->IsValidExpressID(expressID)) return 0;
+    if (!loader->IsValidExpressID(expressID)) return 0;
 
     auto& line = loader->GetLine(expressID);
     return line.ifcType;
@@ -904,7 +903,7 @@ EMSCRIPTEN_BINDINGS(my_module) {
         .field("COORDINATE_TO_ORIGIN", &webifc::utility::LoaderSettings::COORDINATE_TO_ORIGIN)
         .field("CIRCLE_SEGMENTS", &webifc::utility::LoaderSettings::CIRCLE_SEGMENTS)
         .field("TAPE_SIZE", &webifc::utility::LoaderSettings::TAPE_SIZE)
-        .field("NO_CHUNKS", &webifc::utility::LoaderSettings::NO_CHUNKS)
+        .field("MEMORY_LIMIT", &webifc::utility::LoaderSettings::MEMORY_LIMIT)
     ;
 
     emscripten::value_array<std::array<double, 16>>("array_double_16")

src/wasm/web-ifc-test.cpp

@@ -268,7 +268,7 @@ int main()
     // std::string content = ReadFile("C:/Users/qmoya/Desktop/PROGRAMES/VSCODE/IFC.JS/issues/#398 solved/398.ifc");
     // std::string content = ReadFile("C:/Users/qmoya/Desktop/PROGRAMES/VSCODE/IFC.JS/issues/#384/384.ifc");
     // std::string content = ReadFile("C:/Users/qmoya/Desktop/PROGRAMES/VSCODE/IFC.JS/issues/#359/359.ifc");
-    std::string content = ReadFile("C:/Users/qmoya/Desktop/PROGRAMES/VSCODE/IFC.JS/issues/#mep/DM1-3_RIV.ifc");
+    std::string content = ReadFile("/Users/thomasbeach/Downloads/web-ifc-test/data/large.ifc");
 
     webifc::utility::LoaderSettings set;
     set.COORDINATE_TO_ORIGIN = true;

src/web-ifc-api.ts

@@ -60,7 +60,7 @@ export const LINE_END = 9;
  * @property {boolean} OPTIMIZE_PROFILES - If true, the model will return all circular and rectangular profiles as a single geometry.
  * @property {boolean} COORDINATE_TO_ORIGIN - If true, the model will be translated to the origin.
  * @property {number} CIRCLE_SEGMENTS - Number of segments for circles. 
- * @property {number} NO_CHUNKS - The maximum number of chunks of (TAPE_SIZE) that are kept in memory
+ * @property {number} MEMORY_LIMIT - The amount of memory to be reserved for storing IFC data in memory
  * @property {number} TAPE_SIZE - Size of the tape for the loader.
  */
 export interface LoaderSettings {
@@ -72,7 +72,7 @@ export interface LoaderSettings {
     CIRCLE_SEGMENTS_HIGH?: number;
     CIRCLE_SEGMENTS?: number;
     BOOL_ABORT_THRESHOLD?: number;
-    NO_CHUNKS?: number;
+    MEMORY_LIMIT?: number;
     TAPE_SIZE? : number;
 }
 
@@ -215,10 +215,10 @@ export class IfcAPI {
     */
     OpenModels(dataSets: Array<Uint8Array>, settings?: LoaderSettings): Array<number> {
         let s: LoaderSettings = {
-            NO_CHUNKS :  25,
+            MEMORY_LIMIT :  3221225472,
             ...settings
         };
-        s.NO_CHUNKS = s.NO_CHUNKS! / dataSets.length;
+        s.MEMORY_LIMIT = s.MEMORY_LIMIT! / dataSets.length;
         let modelIDs:Array<number> = [];
 
         for (let dataSet of dataSets) modelIDs.push(this.OpenModel(dataSet,s));
@@ -232,7 +232,7 @@ export class IfcAPI {
             COORDINATE_TO_ORIGIN: false,
             CIRCLE_SEGMENTS: 12,
             TAPE_SIZE: 67108864,
-            NO_CHUNKS: 25,
+            MEMORY_LIMIT: 3221225472,
             ...settings
         };
         let deprecated = ['USE_FAST_BOOLS','CIRCLE_SEGMENTS_LOW','CIRCLE_SEGMENTS_MEDIUM','CIRCLE_SEGMENTS_HIGH'];

tests/functional/WebIfcApi.spec.ts

@@ -570,7 +570,7 @@ describe('creating ifc', () => {
 describe('opening large amounts of data', () => {
     test("open a small model but with a heavy memory restriction", () => {
         let s: LoaderSettings = {
-            NO_CHUNKS :  2,
+            MEMORY_LIMIT :  209714,
             TAPE_SIZE : 104857
         };
         const exampleIFCData = fs.readFileSync(path.join(__dirname, '../artifacts/S_Office_Integrated Design Archi.ifc.test'));