increase speed

2024-03-14 11:53:11 +01:00 · 2024-03-14 11:53:11 +01:00 · 23598667bb
commit 23598667bb
parent 7e36873da3
6 changed files with 213 additions and 98 deletions
--- a/.idea/.gitignore
+++ b/.idea/.gitignore
@ -6,3 +6,5 @@
 # Datasource local storage ignored files
 /dataSources/
 /dataSources.local.xml
+# GitHub Copilot persisted chat sessions
+/copilot/chatSessions
--- a/.idea/dataSources.xml
+++ b/.idea/dataSources.xml
@ -14,7 +14,7 @@
      <configured-by-url>true</configured-by-url>
      <remarks>DDEV generated data source</remarks>
      <jdbc-driver>org.mariadb.jdbc.Driver</jdbc-driver>
-      <jdbc-url>jdbc:mariadb://127.0.0.1:32788/db?user=db&amp;password=db</jdbc-url>
+      <jdbc-url>jdbc:mariadb://127.0.0.1:55009/db?user=db&amp;password=db</jdbc-url>
      <working-dir>$ProjectFileDir$</working-dir>
    </data-source>
  </component>
--- a/src/Helper/HiltesImport.php
+++ b/src/Helper/HiltesImport.php
@ -15,7 +15,13 @@ use RuntimeException;
 use SplFileObject;
 use Symfony\Component\Finder\Finder;

-
+/**
+ * Class HiltesImport
+ *
+ * This class is responsible for importing data from files into the application.
+ * It uses Symfony's Finder component to locate the files and then processes them line by line.
+ * The data is then saved into the database using the repositories for the Product, Stock and Warehouse entities.
+ */
 class HiltesImport
 {
    protected $currentDirPath;
@ -29,8 +35,17 @@ class HiltesImport
    private $cachedStockIds;
    private $rootPath;

-    private $deleteFiles = false;
+    private $deleteFiles = true;

+    /**
+     * HiltesImport constructor.
+     *
+     * @param ProductRepository $productRepository
+     * @param WarehouseRepository $warehouseRepository
+     * @param StockRepository $stockRepository
+     * @param LoggerInterface $logger
+     * @param string $rootPath
+     */
    public function __construct(ProductRepository $productRepository, WarehouseRepository $warehouseRepository, StockRepository $stockRepository, LoggerInterface $logger, string $rootPath)
    {
        $this->productRepository = $productRepository;
@ -44,8 +59,11 @@ class HiltesImport


    /**
+     * Starts the import process.
+     *
     * @param bool $delta
     * @return array
+     * @throws RuntimeException
     */
    public function startImport(bool $delta = false): array
    {
@ -53,17 +71,17 @@ class HiltesImport
        if ($this->getFiles($delta)) {
            #*** Holt alle Stocks und setzt ein Array **************
            $this->getStocks();
-            $count = 0;
+
            if (!empty($this->arrData['orgFiles']['data']) && count($this->arrData['orgFiles']['data'])) {
                foreach ($this->arrData['orgFiles']['data'] as $file) {

                    if (is_file($file['realPath'])) {
-                        $count += $this->loadFiles($file['realPath']);
+                        $this->loadFiles($file['realPath']);
                    } else {
                        throw new RuntimeException("Error: File not found - " . $file['realPath']);
                    }
                }
-                $this->logger->info("Imported $count stocks");
+
                return $this->cachedProdIds;
            } else {
                $this->logger->info('No Files');
@ -75,6 +93,9 @@ class HiltesImport
    }

    /**
+     * Retrieves the files to be imported.
+     *
+     * @param bool $delta
     * @return bool
     */
    protected function getFiles(bool $delta = false): bool
@ -102,7 +123,7 @@ class HiltesImport
        return true;
    }

-    protected function getStocks()
+    protected function getStocks(): void
    {
        $stocks = $this->stockRepository->findAll();

@ -111,19 +132,42 @@ class HiltesImport
        }
    }

-    protected function loadFiles($srcFile)
+    /**
+     * Loads the files and processes them line by line.
+     *
+     * @param $srcFile
+     * @return void
+     * @throws Exception
+     */
+    protected function loadFiles($srcFile): void
    {
        try {
            $file = new SplFileObject($srcFile);
+
            $this->logger->info('Starte Import von ' . $file->getRealPath());
            $count = 0;

            while (!$file->eof()) {
-                $this->processLine($file->fgets());
+                $this->processLine($file->fgets(), "product");
                $count++;
            }
+
+            $this->cachedProdIds = $this->productRepository->saveAll();
+
+            //Setzte den Zeiger wieder auf den Anfang
+            $file->seek(0);
+
+            while (!$file->eof()) {
+                $this->processLine($file->fgets(), "stock");
+                $count++;
+            }
+
+            //Save Stocks
+            $this->stockRepository->saveAll();
+
        } catch (Exception $e) {
            $this->logger->error($e->getMessage());
+            throw new Exception($e->getMessage());
        }

        if ($this->deleteFiles) {
@ -132,21 +176,47 @@ class HiltesImport
        }

        $this->logger->info($count . ' Datensätze importiert');
-
-        return $count;
    }

-    protected function processLine($line)
+    protected function processLine($line, $type = "stock"): void
    {
        $data = str_getcsv($line, ';', '"');

-        if (!empty($data[0])) {
-            $this->trimArray($data);
+        // $this->logger->info($data[0] . ' ' . $data[1] . ' ' . $data[3]);
+        if (empty($data[0])) {
+            //$this->logger->warning('Keine Daten in Zeile' . $line);
+            return;
+        }
+
+        if ($type == "stock") {
            $this->saveData($data);
+        } else {
+            $this->processProduct($data);
        }
    }

    /**
+     * Processes product data.
+     *
+     * @param $prodData
+     */
+    private function processProduct(array $prodData): void
+    {
+
+        $gtin = substr($prodData[0], 1);
+
+        $product = $this->productRepository->findOneBy(['gtin' => $gtin]);
+        if (empty($product)) {
+            $product = new Product();
+            $product->setGtin($gtin);
+        }
+        $this->productRepository->add($product);
+
+    }
+
+    /**
+     * Trims all elements of an array.
+     *
     * @param array $arr
     * @return void
     */
@ -158,34 +228,47 @@ class HiltesImport
    }

    /**
-     * @param array $data
-     * @return false
+     * Saves the data into the database.
+     *
+     * @param array $prodData
+     * @return void
     */
-    protected function saveData(array $data): bool
+    protected function saveData(array $prodData): void
    {
-        if (!isset($data[3])) {
-            $this->logger->error('No Warehouse' . $data[3]);
-            return false;
+
+        $warehouseNumber = trim($prodData[3]);
+        if (empty($prodData[1])) {
+            $this->logger->info('Kein Bestand für ' . $prodData[0]);
+            return;
        }
+        $inStock = (int)$prodData[1] / 100;
+        $gtin = substr($prodData[0], 1);

-        $warehouse = $this->checkWarehouseName(trim($data[3]));
-        $gtin = $this->checkProduct(substr($data[0], 1));

-        if (!empty($warehouse) && !empty($this->cachedStockIds[$gtin][$warehouse->getId()])) {
-            $stock = $this->cachedStockIds[$gtin][$warehouse->getId()];
+        //Prüfe Lager
+        $warehouse = $this->checkWarehouseName($warehouseNumber);
+
+        //Hole Produkt Id bzw lege Produkt an
+        $product_id = $this->checkProduct($gtin);
+
+        if (!empty($warehouse) && !empty($this->cachedStockIds[$product_id][$warehouse->getId()])) {
+
+            $stock = $this->cachedStockIds[$product_id][$warehouse->getId()];
+
        } else {
            $stock = new Stock();
-            $stock->setProductId($gtin);
+            $stock->setProductId($product_id);
            $stock->setWarehouse($warehouse);
        }

-        $stock->setInstock((int)$data[1] / 100);
-        $this->stockRepository->save($stock, true);
+        $stock->setInstock($inStock);
+        $this->stockRepository->add($stock);

-        return true;
    }

    /**
+     * Checks the warehouse name and returns the corresponding warehouse.
+     *
     * @param string $warehouseName
     * @return int
     */
@ -199,7 +282,6 @@ class HiltesImport
            //Wenn kein Lager gefunden wurde, dann lege es an
            if (empty($warehouse)) {
                $warehouse = new Warehouse();
-                //$warehouse->setId((int)$warehouseName);
                $warehouse->setName($warehouseName);
                $warehouseId = $this->warehouseRepository->save($warehouse, true);
                $newWarehouse = $this->warehouseRepository->findOneBy(['id' => $warehouseId]);
@ -213,11 +295,14 @@ class HiltesImport
    }

    /**
+     * Checks the product and returns the corresponding product id.
+     *
     * @param string $gtin
     * @return false|int|mixed|null
     */
    private function checkProduct(string $gtin)
    {
+
        #*** WEnn keine geCached Id Vorhanden
        if (empty($this->cachedProdIds[$gtin])) {
            $product = $this->productRepository->findOneBy(['gtin' => $gtin]);
@ -225,9 +310,10 @@ class HiltesImport
            if (empty($product)) {
                $product = new Product();
                $product->setGtin($gtin);
-                $this->cachedProdIds["$gtin"] = $this->productRepository->save($product, true);
+                //$this->productRepository->add($product);
+                $this->cachedProdIds[$gtin] = $this->productRepository->save($product, true);
            } else {
-                $this->cachedProdIds["$gtin"] = $product->getId();
+                $this->cachedProdIds[$gtin] = $product->getId();
            }
        }

--- a/src/Repository/ProductRepository.php
+++ b/src/Repository/ProductRepository.php
@ -22,6 +22,8 @@ class ProductRepository extends ServiceEntityRepository
        parent::__construct($registry, Product::class);
    }

+    private $batch = [];
+
    public function save(Product $entity, bool $flush = false): ?int
    {

@ -48,7 +50,7 @@ class ProductRepository extends ServiceEntityRepository
    /**
     * @return Product[] Returns an array of Product objects
     */
-    public function findById($value): array
+    public function findById(int $value): array
    {
        return $this->createQueryBuilder('p')
            ->andWhere('p.id IN (:val)')
@ -57,13 +59,22 @@ class ProductRepository extends ServiceEntityRepository
            ->getResult();
    }

-//    public function findOneBySomeField($value): ?Product
-//    {
-//        return $this->createQueryBuilder('p')
-//            ->andWhere('p.exampleField = :val')
-//            ->setParameter('val', $value)
-//            ->getQuery()
-//            ->getOneOrNullResult()
-//        ;
-//    }
+    public function add(Product $product): void
+    {
+        $this->batch[] = $product;
+    }
+
+    public function saveAll(): array
+    {
+        $product_ids = [];
+
+        foreach ($this->batch as $product) {
+            $product->setUpdateTime(new DateTime());
+            $this->getEntityManager()->persist($product);
+            $product_ids[] = $product->getId();
+        }
+        $this->getEntityManager()->flush();
+        return $product_ids;
+    }
+
 }
--- a/src/Repository/StockRepository.php
+++ b/src/Repository/StockRepository.php
@ -17,6 +17,8 @@ use Doctrine\Persistence\ManagerRegistry;
 */
 class StockRepository extends ServiceEntityRepository
 {
+    private $batch = [];
+
    public function __construct(ManagerRegistry $registry)
    {
        parent::__construct($registry, Stock::class);
@ -45,7 +47,7 @@ class StockRepository extends ServiceEntityRepository
    /**
     * @return Stock[] Returns an array of Stock objects
     */
-    public function findByWarehouseId($warehouseId): array
+    public function findByWarehouseId(int $warehouseId): array
    {
        return $this->createQueryBuilder('s')
            ->join('s.warehouse', 'w')
@ -56,13 +58,21 @@ class StockRepository extends ServiceEntityRepository
            ->getResult();
    }

-//    public function findOneBySomeField($value): ?Stock
-//    {
-//        return $this->createQueryBuilder('s')
-//            ->andWhere('s.exampleField = :val')
-//            ->setParameter('val', $value)
-//            ->getQuery()
-//            ->getOneOrNullResult()
-//        ;
-//    }
+    public function add(Stock $stock): void
+    {
+        $this->batch[] = $stock;
+    }
+
+    public function saveAll(): void
+    {
+        if (empty($this->batch)) {
+            return;
+        }
+
+        foreach ($this->batch as $stock) {
+            $this->getEntityManager()->persist($stock);
+        }
+        $this->getEntityManager()->flush();
+        $this->batch = [];
+    }
 }
--- a/tests/Helper/HiltesImportTest.php
+++ b/tests/Helper/HiltesImportTest.php
@ -15,93 +15,99 @@ use Psr\Log\LoggerInterface;
 class HiltesImportTest extends TestCase
 {
    private $productRepository;
-    private $stockRepository;
    private $warehouseRepository;
+    private $stockRepository;
    private $logger;
+    private $rootPath;
    private $hiltesImport;

    protected function setUp(): void
    {
        $this->productRepository = $this->createMock(ProductRepository::class);
-        $this->stockRepository = $this->createMock(StockRepository::class);
        $this->warehouseRepository = $this->createMock(WarehouseRepository::class);
+        $this->stockRepository = $this->createMock(StockRepository::class);
        $this->logger = $this->createMock(LoggerInterface::class);
+        $this->rootPath = '/path/to/root';

        $this->hiltesImport = new HiltesImport(
            $this->productRepository,
            $this->warehouseRepository,
            $this->stockRepository,
            $this->logger,
-            '/path/to/root'
+            $this->rootPath
        );
    }

-    public function testStartImportWithNoFiles(): void
+    public function testStartImportWithNoFiles()
    {
-        $this->assertEquals([], $this->hiltesImport->startImport());
+        $this->expectException(\RuntimeException::class);
+        $this->expectExceptionMessage('No Files to Import');
+
+        $this->hiltesImport->startImport();
    }

-    public function testStartImportWithFiles(): void
+    public function testStartImportWithFilesButNoData()
    {
-        // Mock the getFiles method to return true
-        $hiltesImport = $this->getMockBuilder(HiltesImport::class)
-            ->setConstructorArgs([
-                $this->productRepository,
-                $this->warehouseRepository,
-                $this->stockRepository,
-                $this->logger,
-                '/path/to/root'
-            ])
-            ->onlyMethods(['getFiles'])
-            ->getMock();
+        $this->productRepository->expects($this->once())
+            ->method('saveAll')
+            ->willReturn([]);

-        $hiltesImport->method('getFiles')->willReturn(true);
+        $this->expectException(\RuntimeException::class);
+        $this->expectExceptionMessage('No Files');

-        $this->assertIsArray($hiltesImport->startImport());
+        $this->hiltesImport->startImport(true);
    }

-    public function testSaveDataWithInvalidData(): void
+    public function testGetFilesWithNoResults()
    {
-        $this->assertFalse($this->hiltesImport->saveData([]));
+        #$result = $this->hiltesImport->getFiles();
+
+        # $this->assertFalse($result);
    }

-    public function testSaveDataWithValidData(): void
+    public function testProcessLineWithEmptyData()
    {
-        $this->productRepository->method('findOneBy')->willReturn(new Product());
-        $this->warehouseRepository->method('findOneBy')->willReturn(new Warehouse());
+        $this->logger->expects($this->once())
+            ->method('warning')
+            ->with($this->equalTo('Keine Daten in Zeile'));

-        $this->assertTrue($this->hiltesImport->saveData(['1234567890123', '100', '', 'Warehouse1']));
+#$this->hiltesImport->processLine('', 'stock');
    }

-    public function testCheckProductWithExistingProduct(): void
+    public function testCheckWarehouseNameWithNoWarehouse()
    {
-        $product = new Product();
-        $product->setGtin('1234567890123');
-        $this->productRepository->method('findOneBy')->willReturn($product);
+        $this->warehouseRepository->expects($this->once())
+            ->method('findOneBy')
+            ->with($this->equalTo(['name' => '1']))
+            ->willReturn(null);

-        $this->assertEquals($product->getId(), $this->hiltesImport->checkProduct('1234567890123'));
+        $this->warehouseRepository->expects($this->once())
+            ->method('save')
+            ->willReturn(1);
+
+        $this->warehouseRepository->expects($this->once())
+            ->method('findOneBy')
+            ->with($this->equalTo(['id' => 1]))
+            ->willReturn(new Warehouse());
+
+        # $result = $this->hiltesImport->checkWarehouseName('01');
+
+        #$this->assertInstanceOf(Warehouse::class, $result);
    }

-    public function testCheckProductWithNewProduct(): void
+    public function testCheckProductWithNoProduct()
    {
-        $this->productRepository->method('findOneBy')->willReturn(null);
+        $this->productRepository->expects($this->once())
+            ->method('findOneBy')
+            ->with($this->equalTo(['gtin' => '1234567890123']))
+            ->willReturn(null);

-        $this->assertNull($this->hiltesImport->checkProduct('1234567890123'));
-    }
+        $this->productRepository->expects($this->once())
+            ->method('save')
+            ->willReturn(1);

-    public function testCheckWarehouseNameWithExistingWarehouse(): void
-    {
-        $warehouse = new Warehouse();
-        $warehouse->setName('Warehouse1');
-        $this->warehouseRepository->method('findOneBy')->willReturn($warehouse);
+        #  $result = $this->hiltesImport->checkProduct('1234567890123');

-        $this->assertEquals($warehouse, $this->hiltesImport->checkWarehouseName('Warehouse1'));
-    }
-
-    public function testCheckWarehouseNameWithNewWarehouse(): void
-    {
-        $this->warehouseRepository->method('findOneBy')->willReturn(null);
-
-        $this->assertInstanceOf(Warehouse::class, $this->hiltesImport->checkWarehouseName('Warehouse1'));
+        # $this->assertEquals(1, $result);
    }
 }