Add Serilog logging and custom log levels

Added Serilog logging capabilities with `Serilog.Sinks.Seq` and `Serilog.AspNetCore` packages. Introduced `LoggerHelper` and custom log levels (`CustomLogLevel` and `LogLevels`). Updated command handlers and services to use the new logging helper. Modified `GetTMDBData` for async HTTP requests and integrated logging. Enhanced `Cuevana3Services` with logging for pagination and data extraction. Updated Jenkins pipeline to version `0.5.2`. Temporarily disabled authorization for `ScrapPage` endpoint. Configured Serilog in `Program.cs` to read from the application's configuration. Added new files for custom logging levels and helper.

Core.Application/Core.Application.csproj

@@ -16,6 +16,7 @@
     <PackageReference Include="Microsoft.Extensions.Configuration.Abstractions" Version="8.0.0" />
     <PackageReference Include="Newtonsoft.Json" Version="13.0.3" />
     <PackageReference Include="ScrapySharp" Version="3.0.0" />
+    <PackageReference Include="Serilog.Sinks.Seq" Version="9.0.0" />
   </ItemGroup>
 
   <ItemGroup>

Core.Application/Features/Scraping/Cuevana/Cuevana3.ch/Commands/GetAllCuevanaMovies/GetAllCuevanaMoviesCommand.cs

@@ -1,10 +1,14 @@
 using AutoMapper;
+using Core.Application.Helpers.Logger;
+using Core.Application.Helpers.Logs;
 using Core.Application.Helpers.TMDB;
 using Core.Application.Interface.Repositories;
 using Core.Domain.Entities.Movie;
 using Core.Domain.Entities.Relations;
 using Core.Domain.Entities.WebScraping;
 using MediatR;
+using Microsoft.Extensions.Logging;
+using Serilog;
 
 namespace Core.Application.Features.Scraping.Cuevana.Cuevana3.ch.Commands.GetAllCuevanaMovies
 {
@@ -26,17 +30,17 @@ public class GetAllCuevanaMoviesCommandHandler(IMovieWebRepository movieWebRepos
 
         public async Task<bool> Handle(GetAllCuevanaMoviesCommand request, CancellationToken cancellationToken)
         {
+            LoggerHelper.CustomLog(CustomLogLevel.Scraping, "Starting Scraping From Cuevana.biz", LogLevels.Information);
+
             var _cuevanaService = new Services.WebScrapers.MovieESWebsites.Cuevana.Cuevana3Services(1, "https://cuevana.biz");
 
             var pagination = _cuevanaService.GetPagination();
             while (pagination > 0)
             {
-                Console.WriteLine($"Paginacion {pagination}");
-
                 var movies = _cuevanaService.GetMoviesFromPage(pagination);
                 if (movies != null)
                 {
-                    var data = _getTmdbData.GetTMDBId(movies);
+                    var data = await _getTmdbData.GetTMDBIdAsync(movies);
                     List<Movie> uniqueMovies = data.Movies.GroupBy(m => m.TMDBID).Select(g => g.First()).ToList();
                     await _movieRepository.AddAllAsync(await _movieRepository.Exist(uniqueMovies));
                     var movieWeb = await _movieWebRepository.Exist(data.MovieWebDto);
@@ -55,6 +59,8 @@ await _movie_MovieWebRepository.AddAsync(new MovieMovieWeb
 
                         if (movie.Genres != null)
                         {
+                            var Message = $"Adding Genres for {movie.Name}";
+                            LoggerHelper.CustomLog(CustomLogLevel.Scraping, Message, LogLevels.Information);
                             foreach (var genre in movie.Genres)
                             {
                                 var genreId = await _genreRepository.GetIdByTmdbId(genre);

Core.Application/Features/Scraping/PelisPlusLat/Commands/GetPelisPlusLatMovies/GetPelisPlusLatMoviesCommand.cs

@@ -15,10 +15,10 @@ public class GetPelisPlusLatMoviesCommand : IRequest<bool>
     }
     public class GetPelisPlusLatMoviesCommandHandler(
         IScrapPageRepository scrapPageRepository,
-        IMovieWebRepository movieWebRepository,
-        IMovie_MovieWebRepository movie_MovieWebRepository,
-        IMovieRepository movieRepository,
-        GetTmdbData getTmdbData,
+        IMovieWebRepository movieWebRepository, 
+        IMovie_MovieWebRepository movie_MovieWebRepository, 
+        IMovieRepository movieRepository, 
+        GetTmdbData getTmdbData, 
         ILogger<GetPelisPlusLatMoviesCommandHandler> logger,
         IMapper mapper) : IRequestHandler<GetPelisPlusLatMoviesCommand, bool>
     {
@@ -47,7 +47,7 @@ public async Task<bool> Handle(GetPelisPlusLatMoviesCommand request, Cancellatio
                     var movieRaw = PelisPlusLatMovies.GetPelisplushd(i);
                     if (movieRaw != null)
                     {
-                        var data = _getTMDBData.GetTMDBId(movieRaw);
+                        var data = await _getTMDBData.GetTMDBIdAsync(movieRaw);
                         List<Movie> uniqueMovies = data.Movies.GroupBy(m => m.TMDBID).Select(g => g.First()).ToList();
                         await _movieRepository.AddAllAsync(await _movieRepository.Exist(uniqueMovies)); //Movie Added if not exist
                         var movies = await _movieWebRepository.Exist(data.MovieWebDto);

Core.Application/Helpers/Logger/CustomLogLevel.cs

@@ -0,0 +1,13 @@
+using System;
+using System.Collections.Generic;
+using System.Linq;
+using System.Text;
+using System.Threading.Tasks;
+
+namespace Core.Application.Helpers.Logger
+{
+    public enum CustomLogLevel
+    {
+        Scraping,
+    }   
+}

Core.Application/Helpers/Logger/LogLevels.cs

@@ -0,0 +1,17 @@
+using System;
+using System.Collections.Generic;
+using System.Linq;
+using System.Text;
+using System.Threading.Tasks;
+
+namespace Core.Application.Helpers.Logger
+{
+    public enum LogLevels
+    {
+        Information,
+        Warning,
+        Error,
+        Debug,
+        Fatal
+    }
+}

Core.Application/Helpers/Logger/LoggerHelper.cs

@@ -0,0 +1,45 @@
+using Core.Application.Helpers.Logger;
+using Serilog;
+
+namespace Core.Application.Helpers.Logs
+{
+    public static class LoggerHelper
+    {
+        public static void CustomLog(CustomLogLevel customLevel, string customMessage, LogLevels level, string Value = "")
+        {
+            var log = Log.ForContext("CustomLevel", customLevel.ToString());
+
+            if (!string.IsNullOrEmpty(Value))
+            {
+                log = log.ForContext("Data", Value);
+            }
+
+            switch (level)
+            {
+                case LogLevels.Information:
+                    log.Information(customMessage);
+                    break;
+
+                case LogLevels.Warning:
+                    log.Warning(customMessage);
+                    break;
+
+                case LogLevels.Error:
+                    log.Error(customMessage);
+                    break;
+
+                case LogLevels.Debug:
+                    log.Debug(customMessage);
+                    break;
+
+                case LogLevels.Fatal:
+                    log.Fatal(customMessage);
+                    break;
+
+                default:
+                    log.Information(customMessage);
+                    break;
+            }
+        }
+    }
+}

Core.Application/Helpers/TMDB/GetTMDBData.cs

@@ -2,10 +2,12 @@
 using Core.Application.DTOs.Genres;
 using Core.Application.DTOs.Scraping;
 using Core.Application.DTOs.TMDB;
+using Core.Application.Helpers.Logger;
+using Core.Application.Helpers.Logs;
 using Core.Domain.Entities.Movie;
+using Microsoft.AspNetCore.Mvc.RazorPages;
 using Microsoft.Extensions.Configuration;
 using Newtonsoft.Json;
-using System.Net;
 
 namespace Core.Application.Helpers.TMDB
 {
@@ -22,20 +24,30 @@ public GetTmdbData(IConfiguration configuration, IMapper mapper)
             TMDBAPIKEY = Environment.GetEnvironmentVariable("TMDBAPIKey") ?? _configuration["TMDBAPIKey"] ?? throw new ArgumentNullException(nameof(configuration), "TMDBAPIKey is not set.");
         }
 
-        public CheckedList GetTMDBId(List<MovieWebDto> movies)
+        public async Task<CheckedList> GetTMDBIdAsync(List<MovieWebDto> movies)
         {
             List<Movie> CheckData = [];
             List<MovieWebDto> MovieData = [];
+            string Message;
+
             foreach (var data in movies)
             {
                 try
                 {
-                    string TMDBData = new WebClient().DownloadString($"https://api.themoviedb.org/3/search/movie?api_key={TMDBAPIKEY}&language=es-MX&query={data.Name}&include_adult=true");
-                    var result = JsonConvert.DeserializeObject<TmdbResponse>(TMDBData);
+                    Message = $"Getting TMDB data for {data.Name}";
+                    LoggerHelper.CustomLog(CustomLogLevel.Scraping, Message, LogLevels.Information);
+
+                    using var client = new HttpClient();
+                    var response = await client.GetStringAsync($"https://api.themoviedb.org/3/search/movie?api_key={TMDBAPIKEY}&language=es-MX&query={data.Name}&include_adult=true");
+                    var result = JsonConvert.DeserializeObject<TmdbResponse>(response);
                     if (result?.Results != null)
                     {
                         TmdbResult tmdb = result.Results.FirstOrDefault()!;
+
+                        Message = $"TMDB data found for {data.Name}";
+                        LoggerHelper.CustomLog(CustomLogLevel.Scraping, Message, LogLevels.Information);
                         Console.WriteLine("Getting TMDB Data");
+
                         if (tmdb != null)
                         {
                             data.TMDBTempID = tmdb.ID;
@@ -49,6 +61,8 @@ public CheckedList GetTMDBId(List<MovieWebDto> movies)
                 }
                 catch (Exception ex)
                 {
+                    Message = $"Error getting TMDB data for {data.Name}";
+                    LoggerHelper.CustomLog(CustomLogLevel.Scraping, Message, LogLevels.Error, ex.Message);
                     Console.WriteLine(ex);
                 }
             }

Core.Application/Services/WebScrapers/MovieESWebsites/Cuevana/Cuevana3Services.cs

@@ -1,5 +1,8 @@
 using Core.Application.DTOs.Scraping;
+using Core.Application.Helpers.Logger;
+using Core.Application.Helpers.Logs;
 using HtmlAgilityPack;
+using Microsoft.AspNetCore.Mvc.RazorPages;
 using System.Net;
 
 namespace Core.Application.Services.WebScrapers.MovieESWebsites.Cuevana
@@ -60,6 +63,9 @@ public class Cuevana3Services(int DB_WEB_ID, string ORIGINAL_URI)
         /// <returns>The total number of pages for the movies.</returns>
         public virtual int GetPagination()
         {
+            var Message = "Getting Pagination";
+            LoggerHelper.CustomLog(CustomLogLevel.Scraping, Message, LogLevels.Information);
+
             HtmlWeb web = new();
             var htmlDoc = web.Load(ORIGINAL_URI + PagePaginationUri);
             return (int)Convert.ToInt64(htmlDoc.DocumentNode.SelectSingleNode(PagePaginationNode).InnerText);
@@ -72,6 +78,9 @@ public virtual int GetPagination()
         /// <returns>A list of <see cref="MovieWebDto"/> objects containing movie information.</returns>
         public virtual List<MovieWebDto> GetMoviesFromPage(int page)
         {
+            var Message = $"Working on pagination Number {page}";
+            LoggerHelper.CustomLog(CustomLogLevel.Scraping, Message, LogLevels.Information);
+
             List<MovieWebDto> movieList = [];
             HtmlWeb web = new();
             var htmlDoc = web.Load($"{ORIGINAL_URI + PageNumberUri + page}");
@@ -81,7 +90,13 @@ public virtual List<MovieWebDto> GetMoviesFromPage(int page)
             foreach (var node in elements)
             {
                 count++;
-                movieList.Add(GetMovieInfo(node));
+                var MovieInfo = GetMovieInfo(node, page);
+                if(MovieInfo.Name != null)
+                {
+                    movieList.Add(MovieInfo);
+                }
+                Message = $"Movie number {count} parsed";
+                LoggerHelper.CustomLog(CustomLogLevel.Scraping, Message, LogLevels.Information);
                 Console.WriteLine($"Movie {count}");
             }
             return movieList;
@@ -92,26 +107,43 @@ public virtual List<MovieWebDto> GetMoviesFromPage(int page)
         /// </summary>
         /// <param name="node">The HTML node containing the movie information.</param>
         /// <returns>A <see cref="MovieWebDto"/> containing the movie data.</returns>
-        private MovieWebDto GetMovieInfo(HtmlNode node)
+        private MovieWebDto GetMovieInfo(HtmlNode Node, int Page)
         {
-            var movieName = node.SelectSingleNode(GetMovieName).InnerText;
-            var movieUrl = node.SelectSingleNode(GetMovieUrl).GetAttributeValue("href", "ERROR");
+            var movieName = Node.SelectSingleNode(GetMovieName)?.InnerText;
+            var movieUrl = Node.SelectSingleNode(GetMovieUrl)?.GetAttributeValue("href", "ERROR");
+            string Message;
+
+            if (movieName == null || movieUrl == null)
+            {
+                Message = $"Name or Uri Dont found in page {Page}";
+                LoggerHelper.CustomLog(CustomLogLevel.Scraping, Message, LogLevels.Error, Node.InnerHtml);
+
+                return new MovieWebDto
+                {
+                    Name = string.Empty,
+                    Img = string.Empty,
+                    Url = string.Empty,
+                    Overview = string.Empty,
+                    ScrapPageID = DB_WEB_ID
+                };
+            }
             var movieImage = WebUtility.UrlDecode(
-                node.SelectSingleNode(GetMovieImage)?
+                Node.SelectSingleNode(GetMovieImage)?
                     .GetAttributeValue("src", "ERROR")
                     .Replace(ReplaceMovieUri, "")
             );
 
-            var movieData = new MovieWebDto
+            Message = $"Data Taken from movie {movieName}";
+            LoggerHelper.CustomLog(CustomLogLevel.Scraping, Message, LogLevels.Information);
+
+            return new MovieWebDto
             {
                 Name = movieName,
                 Img = movieImage,
                 Url = movieUrl,
                 Overview = GetOverView(movieUrl),
                 ScrapPageID = DB_WEB_ID
             };
-
-            return movieData;
         }
 
         /// <summary>
@@ -121,19 +153,24 @@ private MovieWebDto GetMovieInfo(HtmlNode node)
         /// <returns>The movie overview or "Error" if the description cannot be retrieved.</returns>
         private string GetOverView(string uri)
         {
+            var Message = $"Getting overview From Movie {uri}";
+            LoggerHelper.CustomLog(CustomLogLevel.Scraping, Message, LogLevels.Information);
+
             string? node;
+            HtmlWeb web = new();
+            var htmlDoc = web.Load(ORIGINAL_URI + uri);
 
             try
             {
-                HtmlWeb web = new();
-                var htmlDoc = web.Load(ORIGINAL_URI + uri);
                 node = htmlDoc.DocumentNode.SelectSingleNode(GetMovieDescription)?.InnerText;
             }
             catch
             {
+                Message = $"Error Getting Overview for {uri}";
+                LoggerHelper.CustomLog(CustomLogLevel.Scraping, Message, LogLevels.Error, htmlDoc.DocumentNode.InnerHtml);
                 node = "Error";
             }
-            return node;
+            return node!;
         }
     }
 }

Jenkinsfiles/TestJenkins

@@ -1,7 +1,7 @@
 pipeline {
     agent { label 'Africa' }
     environment {
-        VERSION = "0.5.1"
+        VERSION = "0.5.2"
         FTP_SERVER = credentials('abreu_fpt_host')
         NUGET_SOURCE = credentials('private_nuget_uri')
         SONARQUBE_EXCLUDE_FILES = "**/*.xml,**/*.json,**/*.html,**/*.css,**/*.js,**/Program.cs,**/Startup.cs,**/Migrations/**"

Presentation.KuhakuCentral/Controllers/V1/WebScrapingModule/WebScrapingModuleController.cs

@@ -7,14 +7,16 @@
 using Swashbuckle.AspNetCore.Annotations;
 using System.Net;
 using System.Net.Mime;
+using Serilog;
+using Core.Application.Helpers.Logs;
 
 namespace KuhakuCentral.Controllers.V1.WebScrapingModule
 {
     public class WebScrapingModuleController : BaseApi
     {
 
         [HttpGet("ScrapPage")]
-        [Authorize(Roles = "Owner")]
+        //[Authorize(Roles = "Owner")]
         [Consumes(MediaTypeNames.Application.Json)]
         [ProducesResponseType(StatusCodes.Status404NotFound)]
         [ProducesResponseType(StatusCodes.Status500InternalServerError)]

Presentation.KuhakuCentral/KuhakuCentral.csproj

@@ -20,6 +20,8 @@
     </PackageReference>
     <PackageReference Include="MimeKit" Version="4.9.0" />
     <PackageReference Include="Scalar.AspNetCore" Version="2.0.12" />
+    <PackageReference Include="Serilog.AspNetCore" Version="9.0.0" />
+    <PackageReference Include="Serilog.Sinks.Seq" Version="9.0.0" />
     <PackageReference Include="Swashbuckle.AspNetCore" Version="7.1.0" />
     <PackageReference Include="Swashbuckle.AspNetCore.Annotations" Version="7.1.0" />
   </ItemGroup>

Presentation.KuhakuCentral/Program.cs

@@ -5,6 +5,7 @@
 using Microsoft.AspNetCore.HttpOverrides;
 using Microsoft.AspNetCore.Mvc;
 using Scalar.AspNetCore;
+using Serilog;
 using Swashbuckle.AspNetCore.SwaggerUI;
 
 var builder = WebApplication.CreateBuilder(args);
@@ -21,6 +22,9 @@
 builder.Services.AddAuthentication();
 builder.Services.AddIdentityInfrastructure(builder.Configuration);
 
+builder.Host.UseSerilog((context, config) =>
+    config.ReadFrom.Configuration(context.Configuration));
+
 //new
 builder.Services.AddControllers(options =>
 {