Skip to content

Commit

Permalink
uploaded item old foreign keys dropped
Browse files Browse the repository at this point in the history
  • Loading branch information
@pavel-zhur
pavel-zhur committed Sep 27, 2024
1 parent 774a339 commit 7d0d778
Show file tree
Hide file tree
Showing 8 changed files with 935 additions and 63 deletions.
12 changes: 3 additions & 9 deletions OneShelf.Videos/OneShelf.Videos.BusinessLogic/Services/ExifService.cs
View file
Original file line number Diff line number Diff line change
@@ -1,22 +1,16 @@
using System.Globalization;
using ExifLibrary;
using OneShelf.Common;
using ExifLibrary;

namespace OneShelf.Videos.BusinessLogic.Services;

public class ExifService
{
public async Task<DateTime> SetExifTimestamp(string path, string tempFileName)
public async Task SetExifTimestamp(string path, string tempFileName, DateTime timestamp)
{
var image = await ImageFile.FromFileAsync(path);
if (image.Errors.Any()) throw new($"Some image errors, {path}.");
if (image.Properties.Contains(ExifTag.DateTime)) throw new($"The image contains a datetime, {path}.");

var dateTime = DateTime.ParseExact(Path.GetFileNameWithoutExtension(path).SelectSingle(x => x.Substring(x.IndexOf('@') + 1)), "dd-MM-yyyy_HH-mm-ss", CultureInfo.InvariantCulture, DateTimeStyles.AssumeUniversal);

image.Properties.Set(ExifTag.DateTime, dateTime);
image.Properties.Set(ExifTag.DateTime, timestamp);
await image.SaveAsync(tempFileName);

return dateTime;
}
}
56 changes: 32 additions & 24 deletions OneShelf.Videos/OneShelf.Videos.BusinessLogic/Services/Service1.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -23,31 +23,37 @@ public Service1(IOptions<VideosOptions> options, VideosDatabase videosDatabase,
_logger = logger;
}

public async Task<List<(long chatId, int messageId, string path, DateTime publishedOn)>> GetExport1()
public async Task<List<(int mediaId, string path, DateTime publishedOn)>> GetExport1()
{
var messages = await _videosDatabase.StaticMessages
.Where(x => x.SelectedType == StaticMessageSelectedType.Photo)
.Include(x => x.StaticChat)
var messages = await _videosDatabase.Mediae
.Where(x => x.StaticMessage != null)
.Where(x => x.StaticMessage!.SelectedType == StaticMessageSelectedType.Photo)
.Where(x => x.StaticMessage!.Media != null)
.Where(x => x.UploadedItem == null)
.Include(x => x.StaticMessage)
.ThenInclude(x => x.StaticChat)
.ThenInclude(x => x.StaticChatFolder)
.Where(x => !_videosDatabase.UploadedItems.Any(y => y.StaticChatId == x.StaticChatId && y.StaticMessageId == x.Id))
.ToListAsync();

return messages
.Select(x => (x.StaticChat.Id, x.Id, Path.Combine(x.StaticChat.StaticChatFolder.Root, x.Photo!), x.Date))
.Select(x => (x.Id, Path.Combine(x.StaticMessage!.StaticChat.StaticChatFolder.Root, x.StaticMessage.Photo!), x.StaticMessage.Date))
.ToList();
}

public async Task<List<(long chatId, int messageId, string path, DateTime publishedOn)>> GetExport2()
public async Task<List<(int mediaId, string path, DateTime publishedOn)>> GetExport2()
{
var messages = await _videosDatabase.StaticMessages
.Where(x => x.SelectedType == StaticMessageSelectedType.Video)
.Include(x => x.StaticChat)
var messages = await _videosDatabase.Mediae
.Where(x => x.StaticMessage != null)
.Where(x => x.StaticMessage!.SelectedType == StaticMessageSelectedType.Video)
.Where(x => x.StaticMessage!.Media != null)
.Where(x => x.UploadedItem == null)
.Include(x => x.StaticMessage)
.ThenInclude(x => x.StaticChat)
.ThenInclude(x => x.StaticChatFolder)
.Where(x => !_videosDatabase.UploadedItems.Any(y => y.StaticChatId == x.StaticChatId && y.StaticMessageId == x.Id))
.ToListAsync();

return messages
.Select(x => (x.StaticChat.Id, x.Id, Path.Combine(x.StaticChat.StaticChatFolder.Root, x.File!), x.Date))
.Select(x => (x.Id, Path.Combine(x.StaticMessage!.StaticChat.StaticChatFolder.Root, x.StaticMessage.File!), x.StaticMessage.Date))
.ToList();
}

Expand Down Expand Up @@ -93,32 +99,32 @@ public async Task SaveMessages()
_logger.LogInformation("Saved.");
}

public async Task<List<(int albumId, string title, List<(string? mediaItemId, long chatId, int messageId)> items)>> GetAlbums()
public async Task<List<(int albumId, string title, List<(string mediaItemId, int mediaId)> items)>> GetAlbums()
{
var albums = await _videosDatabase.Albums
.Where(x => x.UploadedAlbum == null)
.Include(x => x.Constraints)
.ThenInclude(x => x.StaticTopic)
.ToListAsync();

var messages = (await _videosDatabase.StaticMessages
.Where(x => x.SelectedType.HasValue && x.StaticTopicId.HasValue)
var messages = (await _videosDatabase.Mediae
.Where(x => x.StaticMessage != null)
.Where(x => x.StaticMessage!.SelectedType.HasValue && x.StaticMessage.StaticTopicId.HasValue)
.Select(m => new
{
TopicId = m.StaticTopicId,
ChatId = m.StaticChatId,
m.Width,
m.Height,
m.Date,
TopicId = m.StaticMessage!.StaticTopicId,
m.StaticMessage.Width,
m.StaticMessage.Height,
m.StaticMessage.Date,
m.StaticMessage.SelectedType,
m.Id,
m.SelectedType,
})
.ToListAsync())
.ToLookup(x => x.TopicId!.Value);

var uploadedItems = await _videosDatabase.UploadedItems
.Where(i => _videosDatabase.InventoryItems.Any(j => j.Id == i.MediaItemId && (j.IsPhoto || j.MediaMetadataVideoStatus == "READY")))
.ToDictionaryAsync(x => new { ChatId = x.StaticChatId, MessageId = x.StaticMessageId, }, x => x.MediaItemId);
.ToDictionaryAsync(x => x.MediaId, x => x.MediaItemId);

return albums
.Select(a => (a.Id, a.Title,
Expand All @@ -132,9 +138,11 @@ public async Task SaveMessages()
if (c.Before.HasValue && m.Date > c.Before) return false;
return true;
}))
.Select(x => new { x.ChatId, MessageId = x.Id })
.Select(x => x.Id)
.Distinct()
.Select(x => (uploadedItems.GetValueOrDefault(x), x.ChatId, x.MessageId))
.Select(x => (mediaItemId: uploadedItems.GetValueOrDefault(x), x))
.Where(x => x.mediaItemId != null)
.Select(x => (x.mediaItemId!, x.x))
.ToList()))
.ToList();
}
Expand Down
39 changes: 21 additions & 18 deletions OneShelf.Videos/OneShelf.Videos.BusinessLogic/Services/Service2.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,7 @@
using OneShelf.Common;
using OneShelf.Videos.Database;
using OneShelf.Videos.Database.Models;
using System.Globalization;

namespace OneShelf.Videos.BusinessLogic.Services;

Expand Down Expand Up @@ -68,15 +69,15 @@ public async Task SaveInventory()
_logger.LogInformation("Saved.");
}

public async Task UploadPhotos(List<(long chatId, int messageId, string path, DateTime publishedOn)> items)
public async Task UploadPhotos(List<(int mediaId, string path, DateTime publishedOn)> items)
{
await _extendedGooglePhotosService.LoginAsync();

var itemsByKey = items.ToDictionary(x => (x.chatId, x.messageId));
var fileNameTimestamps = new Dictionary<(long chatId, int messageId), DateTime>();
var itemsByKey = items.ToDictionary(x => x.mediaId);
var fileNameTimestamps = new Dictionary<int, DateTime>();
var result = await _extendedGooglePhotosService.UploadMultiple(
items
.Select(x => ((x.chatId, x.messageId), x.path, (string?)null))
.Select(x => (x.mediaId, x.path, (string?)null))
.ToList(),
newItems => AddToDatabase(itemsByKey, newItems, fileNameTimestamps),
async (x, i) =>
Expand All @@ -88,11 +89,13 @@ public async Task UploadPhotos(List<(long chatId, int messageId, string path, Da
}

Directory.CreateDirectory(tempDirectory);
var tempFileName = Path.Combine(Path.GetTempPath(), i.ToString(), Path.GetFileName(itemsByKey[x].path));
var timestampFromFile = await _exifService.SetExifTimestamp(itemsByKey[x].path, tempFileName);
var fileName = itemsByKey[x].path;
var tempFileName = Path.Combine(Path.GetTempPath(), i.ToString(), Path.GetFileName(fileName));
var exifTimestamp = DateTime.ParseExact(Path.GetFileNameWithoutExtension(fileName).SelectSingle(x => x.Substring(x.IndexOf('@') + 1)), "dd-MM-yyyy_HH-mm-ss", CultureInfo.InvariantCulture, DateTimeStyles.AssumeUniversal);
await _exifService.SetExifTimestamp(fileName, tempFileName, exifTimestamp);
lock (fileNameTimestamps)
{
fileNameTimestamps[x] = timestampFromFile;
fileNameTimestamps[x] = exifTimestamp;
}

return tempFileName;
Expand All @@ -102,19 +105,19 @@ public async Task UploadPhotos(List<(long chatId, int messageId, string path, Da
Console.WriteLine($"started: {items.Count}, finished: {result.Count}");
}

public async Task UploadVideos(List<(long chatId, int messageId, string path, DateTime publishedOn)> items)
public async Task UploadVideos(List<(int mediaId, string path, DateTime publishedOn)> items)
{
await _extendedGooglePhotosService.LoginAsync();

var added = (await _videosDatabase.UploadedItems.Select(x => new { ChatId = x.StaticChatId, MessageId = x.StaticMessageId }).ToListAsync()).ToHashSet();
var added = (await _videosDatabase.UploadedItems.Select(x => x.MediaId).ToListAsync()).ToHashSet();
Console.WriteLine($"initial items: {items.Count}");
items = items.Where(x => !added.Contains(new { ChatId = x.chatId, MessageId = x.messageId })).ToList();
items = items.Where(x => !added.Contains(x.mediaId)).ToList();
Console.WriteLine($"remaining items: {items.Count}");

var itemsByKey = items.ToDictionary(x => (x.chatId, x.messageId));
var itemsByKey = items.ToDictionary(x => x.mediaId);
var result = await _extendedGooglePhotosService.UploadMultiple(
items
.Select(x => ((x.chatId, x.messageId), x.path,
.Select(x => (x.mediaId, x.path,
//$"chatId = {x.chatId}, messageId = {x.messageId}, published on = {x.publishedOn}, filename = {Path.GetFileName(x.path)}"
(string?)null))
.ToList(),
Expand All @@ -126,22 +129,22 @@ public async Task UploadVideos(List<(long chatId, int messageId, string path, Da
}

private async Task AddToDatabase(
Dictionary<(long chatId, int messageId), (long chatId, int messageId, string path, DateTime publishedOn)> items,
Dictionary<(long chatId, int messageId), NewMediaItemResult> newItems,
Dictionary<(long chatId, int messageId), DateTime>? fileNameTimestamps = null)
Dictionary<int, (int mediaId, string path, DateTime publishedOn)> items,
Dictionary<int, NewMediaItemResult> newItems,
Dictionary<int, DateTime>? fileNameTimestamps = null)
{
_videosDatabaseOperations.AddItems(newItems.Select(i => items[i.Key].SelectSingle(x => (x.chatId, x.messageId, x.path, x.publishedOn, result: i.Value, fileNameTimestamp: fileNameTimestamps?[i.Key]))));
_videosDatabaseOperations.AddItems(newItems.Select(i => items[i.Key].SelectSingle(x => (x.mediaId, x.path, x.publishedOn, result: i.Value, fileNameTimestamp: fileNameTimestamps?[i.Key]))));
await _videosDatabase.SaveChangesAsync();
}

public async Task CreateAlbums(List<(int albumId, string title, List<(string? mediaItemId, long chatId, int messageId)> items)> albums)
public async Task CreateAlbums(List<(int albumId, string title, List<(string mediaItemId, int mediaId)> items)> albums)
{
await _extendedGooglePhotosService.LoginAsync();
foreach (var (albumId, title, items) in albums)
{
_logger.LogInformation("{title} uploading...", title);
var googleAlbum = await _extendedGooglePhotosService.CreateAlbumAsync(title);
await _extendedGooglePhotosService.AddMediaItemsToAlbumWithRetryAsync(googleAlbum!.id, items.Where(x => x.mediaItemId != null).Select(x => x.mediaItemId!).ToList());
await _extendedGooglePhotosService.AddMediaItemsToAlbumWithRetryAsync(googleAlbum!.id, items.Select(x => x.mediaItemId).ToList());

_videosDatabase.UploadedAlbums.Add(new()
{
Expand Down
5 changes: 2 additions & 3 deletions OneShelf.Videos/OneShelf.Videos.BusinessLogic/Services/VideosDatabaseOperations.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -7,13 +7,12 @@ namespace OneShelf.Videos.BusinessLogic.Services;

public class VideosDatabaseOperations(VideosDatabase videosDatabase)
{
public void AddItems(IEnumerable<(long chatId, int messageId, string path, DateTime publishedOn, NewMediaItemResult result, DateTime? fileNameTimestamp)> items)
public void AddItems(IEnumerable<(int mediaId, string path, DateTime publishedOn, NewMediaItemResult result, DateTime? fileNameTimestamp)> items)
{
videosDatabase.UploadedItems.AddRange(items.Select(i => new UploadedItem
{
CreatedOn = DateTime.Now,
StaticChatId = i.chatId,
StaticMessageId = i.messageId,
MediaId = i.mediaId,
TelegramPublishedOn = i.publishedOn,
Status = i.result.status.status,
StatusCode = i.result.status.code,
Expand Down
Loading

0 comments on commit 7d0d778

Please sign in to comment.