zhaolei/FileSqlServerSync
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

feat: 完成发布流程

Browse source
This commit is contained in:
zerlei 2024-09-22 13:27:52 +08:00
parent 63452fe1ab
commit 9c9ffdd77e
11 changed files with 454 additions and 52 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

44
Server/Common/Config.cs
View file

@ -16,9 +16,40 @@ public class DirFileConfig
/// 除此外全部忽略，最高优先级，若有值，ExcludeFiles 将被忽略，它是根目录的相对路径
/// </summary>
public List<string>? CherryPicks { get; set; }
///
public Dir? DirInfo { get; set; }
}
public class MSSqlConfig
{
/// <summary>
/// 数据库地址
/// </summary>
public required string ServerName { get; set; }
/// <summary>
/// db名称
/// </summary>
public required string DatebaseName { get; set; }
/// <summary>
/// 用户
/// </summary>
public required string User { get; set; }
/// <summary>
/// 密码
/// </summary>
public required string Password { get; set; }
/// <summary>
/// 通常是：True
/// </summary>
public required string TrustServerCertificate { get; set; }
/// <summary>
/// 同步数据的表格 ！！！ 通常是 dbo.TableName !!! 注意dbo.
/// </summary>
public List<string>? SyncTablesData{get;set;}
}
public class Config
{
/// <summary>
@ -44,19 +75,14 @@ public class Config
public required bool IsDeployDb { get; set; }
/// <summary>
/// 源数据库连接字符串(ip地址相对LocalServer)
/// 源数据库连接(ip地址相对LocalServer)
/// </summary>
public required string SrcDbConnection { get; set; }
public required MSSqlConfig SrcDb { get; set; }
/// <summary>
/// 目标数据库连接字符串(ip地址相对RemoteServer)
/// 目标数据库(ip地址相对RemoteServer)
/// </summary>
public required string DstDbConnection { get; set; }
/// <summary>
/// 同步的表
/// </summary>
public required List<string>? SyncDataTables { get; set; }
public required MSSqlConfig DstDb { get; set; }
/// <summary>
/// 是否发布项目

32
Server/Common/ConnectPipeline.cs
View file

@ -27,6 +27,7 @@ public abstract class AbsPipeLine(bool isAES)
/// <returns></returns>
public abstract Task SendMsg(SyncMsg msg);
public abstract Task UploadFile(string filePath, string url, Func<double, bool> progressCb);
protected readonly bool IsAES = isAES;
}
@ -35,6 +36,37 @@ public class WebSocPipeLine<TSocket>(TSocket socket, bool isAES) : AbsPipeLine(i
{
public readonly TSocket Socket = socket;
public override async Task UploadFile(
string filePath,
string url,
Func<double, bool> progressCb
)
{
if (Socket is HttpClient)
{
using var client = new HttpClient();
using var content = new MultipartFormDataContent();
using var fileStream = new FileStream(filePath, FileMode.Open);
var progress = new Progress<double>(
(current) =>
{
progressCb(current);
}
);
var fileContent = new ProgressStreamContent(fileStream, progress);
content.Add(fileContent, "file", Path.GetFileName(filePath));
var it = await client.PostAsync(url, content);
if (it.StatusCode != System.Net.HttpStatusCode.OK)
{
throw new Exception(it.Content.ReadAsStringAsync().Result);
}
}
else
{
throw new NotSupportedException("只支持HttpClient!");
}
}
public override async IAsyncEnumerable<int> Work(Func<byte[], bool> receiveCb, string addr = "")
{
if (Socket is ClientWebSocket CSocket)

2
Server/Common/Dir.cs
View file

@ -60,7 +60,7 @@ public class Dir(string path, List<AFileOrDir>? children = null, NextOpType? nex
}
/// <summary>
/// clone
/// clone,不克隆文件
/// </summary>
/// <param name="optype"></param>
/// <param name="IsResetNextOpType"></param>

67
Server/Common/FileDirOp.cs
View file

@ -27,7 +27,7 @@ public abstract class FileDirOpStra
/// 文件目录打包
/// </summary>
/// <param name="dstRootPath"></param>
public class FileDirOpForPack(string srcRootPath, string dstRootPath, string syncId = "")
public class FileDirOpForPack(string srcRootPath, string dstRootPath)
: FileDirOpStra
{
/// <summary>
@ -40,14 +40,10 @@ public class FileDirOpForPack(string srcRootPath, string dstRootPath, string syn
/// </summary>
public readonly string SrcRootPath = srcRootPath;
public readonly string SyncId = string.IsNullOrEmpty(syncId)
? Guid.NewGuid().ToString()
: syncId;
/// <summary>
/// 最终完成时的压缩
/// </summary>
public void FinallyCompress()
public static void FinallyCompress(string dstPath, string Id)
{
static List<string> GetFilesResus(string dirPath)
{
@ -65,9 +61,9 @@ public class FileDirOpForPack(string srcRootPath, string dstRootPath, string syn
}
return files;
}
var fileNames = GetFilesResus(SrcRootPath);
var OuptPutFile = Path.GetDirectoryName(DstRootPath) + $"/{SyncId}.zip";
using FileStream fsOut = new(OuptPutFile, FileMode.Create);
var fileNames = GetFilesResus(dstPath);
var OuptPutFile = Path.GetDirectoryName(dstPath) + $"/{Id}.zip";
using FileStream fsOut = new(OuptPutFile, System.IO.FileMode.Create);
using ZipOutputStream zipStream = new(fsOut);
{
zipStream.SetLevel(9); // 设置压缩级别
@ -78,7 +74,7 @@ public class FileDirOpForPack(string srcRootPath, string dstRootPath, string syn
{
// Using GetFileName makes the result compatible with XP
// as the resulting path is not absolute.
var entry = new ZipEntry(file.Replace(SrcRootPath, ""));
var entry = new ZipEntry(file.Replace(dstPath, ""));
// Setup the entry data as required.
@ -170,15 +166,15 @@ public class FileDirOpForPack(string srcRootPath, string dstRootPath, string syn
public override void DirDel(Dir dir, bool IsRecursion = true) { }
}
public class FileDirOpForUnpack(string srcRootPath, string dstRootPath, string syncId)
public class FileDirOpForUnpack(string srcRootPath, string dstRootPath)
: FileDirOpStra
{
/// <summary>
/// 解压缩,必须首先调用
/// </summary>
public void FirstUnComparess()
public static void FirstUnComparess(string dstPath, string Id)
{
string zipFilePath = $"{SrcRootPath}/{SyncId}.zip";
string zipFilePath = $"{dstPath}/{Id}.zip";
using (ZipInputStream s = new ZipInputStream(System.IO.File.OpenRead(zipFilePath)))
{
@ -189,7 +185,7 @@ public class FileDirOpForUnpack(string srcRootPath, string dstRootPath, string s
Console.WriteLine(theEntry.Name);
string directoryName =
DstRootPath + $"/{SyncId}/" + Path.GetDirectoryName(theEntry.Name)
dstPath + $"/{Id}/" + Path.GetDirectoryName(theEntry.Name)
?? throw new NullReferenceException("无法得到父文件目录！");
string fileName = Path.GetFileName(theEntry.Name);
@ -198,7 +194,6 @@ public class FileDirOpForUnpack(string srcRootPath, string dstRootPath, string s
{
Directory.CreateDirectory(directoryName);
}
if (fileName != String.Empty)
{
using (
@ -227,8 +222,6 @@ public class FileDirOpForUnpack(string srcRootPath, string dstRootPath, string s
}
}
public readonly string SyncId = syncId;
/// <summary>
/// 目标根目录
/// </summary>
@ -244,27 +237,57 @@ public class FileDirOpForUnpack(string srcRootPath, string dstRootPath, string s
/// </summary>
public override void FileCreate(string absolutePath, DateTime mtime)
{
throw new NotImplementedException();
var srcPath = absolutePath.Replace(DstRootPath, SrcRootPath);
var dstDirPath =
Path.GetDirectoryName(absolutePath) ?? throw new NullReferenceException("父路径不存在！");
if (!Directory.Exists(dstDirPath))
{
Directory.CreateDirectory(dstDirPath);
}
//文件时间不会更改
System.IO.File.Copy(srcPath, absolutePath, true);
}
public override void DirCreate(Dir dir, bool IsRecursion = true)
{
throw new NotImplementedException();
// var srcPath = dir.FormatedPath.Replace(DstRootPath, SrcRootPath);
var dstDirPath =
Path.GetDirectoryName(dir.FormatedPath) ?? throw new NullReferenceException("父路径不存在！");
if (!Directory.Exists(dstDirPath))
{
Directory.CreateDirectory(dstDirPath);
}
if (IsRecursion)
{
foreach (var c in dir.Children)
{
if (c is Dir d)
{
this.DirCreate(d, true);
}
else if (c is File f)
{
this.FileCreate(f.FormatedPath, f.MTime);
}
}
}
}
public override void FileModify(string absolutePath, DateTime mtime)
{
throw new NotImplementedException();
this.FileCreate(absolutePath,mtime);
}
public override void FileDel(string absolutePath)
{
throw new NotImplementedException();
System.IO.File.Delete(absolutePath);
}
public override void DirDel(Dir dir, bool IsRecursion = true)
{
throw new NotImplementedException();
System.IO.Directory.Delete(dir.FormatedPath,IsRecursion);
}
}

29
Server/Common/ProgressStreamContent.cs Normal file
View file

@ -0,0 +1,29 @@
using System.Net;
namespace Common;
public class ProgressStreamContent(Stream stream_, IProgress<double> progress)
: StreamContent(stream_, 4096)
{
private readonly Stream FileStream = stream_;
private readonly int BufferSize = 4096;
private readonly IProgress<double> Progress = progress;
protected override async Task SerializeToStreamAsync(
Stream stream,
TransportContext? context = null
)
{
var buffer = new byte[BufferSize];
long totalBytesRead = 0;
long totalBytes = FileStream.Length;
int bytesRead;
while ((bytesRead = await FileStream.ReadAsync(buffer.AsMemory())) != 0)
{
await stream.WriteAsync(buffer.AsMemory(0, bytesRead));
totalBytesRead += bytesRead;
Progress.Report((double)totalBytesRead / totalBytes);
}
}
}

158
Server/LocalServer/StateHelper.cs
View file

@ -2,6 +2,7 @@ using System.Diagnostics;
using System.Runtime.InteropServices;
using System.Text.Json;
using Common;
using Microsoft.AspNetCore.Mvc;
namespace LocalServer;
@ -203,16 +204,163 @@ public class DiffFileAndPackHelper(LocalSyncServer context)
var PackOp = new FileDirOpForPack(
Context.NotNullSyncConfig.LocalRootPath,
Context.NotNullSyncConfig.RemoteRootPath
+ "/"
+ Context.NotNullSyncConfig.Id.ToString(),
LocalSyncServer.TempRootFile + "/" + Context.NotNullSyncConfig.Id.ToString(),
Context.NotNullSyncConfig.Id.ToString()
);
Context.NotNullSyncConfig.DirFileConfigs.ForEach(e =>
{
if (e.DirInfo != null)
{
e.DirInfo.ResetRootPath(
Context.NotNullSyncConfig.RemoteRootPath,
Context.NotNullSyncConfig.LocalRootPath
);
e.DirInfo.WriteByThisInfo(PackOp);
}
});
var n = new DeployMSSqlHelper(Context);
n.PackSqlServerProcess();
Context.StateHelper = n;
}
}
public class DeployMSSqlHelper(LocalSyncServer context)
: StateHelpBase(context, SyncProcessStep.PackSqlServer)
{
private void PackAndSwitchNext()
{
FileDirOpForPack.FinallyCompress(
LocalSyncServer.TempRootFile + "/" + Context.NotNullSyncConfig.Id.ToString(),
Context.NotNullSyncConfig.Id.ToString()
);
var h = new UploadPackedHelper(Context);
Context.StateHelper = h;
h.UpLoadPackedFile();
}
public void PackSqlServerProcess()
{
if (Context.NotNullSyncConfig.IsDeployDb == false)
{
Context.LocalPipe.SendMsg(CreateMsg("配置为不发布数据库跳过此步骤")).Wait();
PackAndSwitchNext();
}
else
{
if (RuntimeInformation.IsOSPlatform(OSPlatform.Windows))
{
var arguments =
$"SqlPackage /Action:Extract /TargetFile:{LocalSyncServer.TempRootFile}/{Context.NotNullSyncConfig.Id.ToString()}/{Context.NotNullSyncConfig.Id.ToString()}.dacpac "
+ $"/DiagnosticsFile:{LocalSyncServer.TempRootFile}/{Context.NotNullSyncConfig.Id.ToString()}/{Context.NotNullSyncConfig.Id.ToString()}.log "
+ $"/p:ExtractAllTableData=false /p:VerifyExtraction=true /SourceServerName:{Context.NotNullSyncConfig.SrcDb.ServerName}"
+ $"/SourceDatabaseName:{Context.NotNullSyncConfig.SrcDb.DatebaseName} /SourceUser:{Context.NotNullSyncConfig.SrcDb.User} "
+ $"/SourcePassword:{Context.NotNullSyncConfig.SrcDb.Password} /SourceTrustServerCertificate:{Context.NotNullSyncConfig.SrcDb.TrustServerCertificate} "
+ $"/p:ExtractReferencedServerScopedElements=False /p:IgnoreUserLoginMappings=True /p:IgnorePermissions=True ";
if (Context.NotNullSyncConfig.SrcDb.SyncTablesData != null)
{
foreach (var t in Context.NotNullSyncConfig.SrcDb.SyncTablesData)
{
arguments += $" /p:TableData={t}";
}
}
ProcessStartInfo startInfo =
new()
{
FileName = "cmd.exe", // The command to execute (can be any command line tool)
Arguments = arguments,
// The arguments to pass to the command (e.g., list directory contents)
RedirectStandardOutput = true, // Redirect the standard output to a string
UseShellExecute = false, // Do not use the shell to execute the command
CreateNoWindow = true // Do not create a new window for the command
};
using Process process = new() { StartInfo = startInfo };
// Start the process
process.Start();
// Read the output from the process
string output = process.StandardOutput.ReadToEnd();
// Wait for the process to exit
process.WaitForExit();
if (process.ExitCode == 0)
{
Context.LocalPipe.SendMsg(CreateMsg("数据库打包成功！")).Wait();
PackAndSwitchNext();
}
else
{
Context.LocalPipe.SendMsg(CreateErrMsg(output)).Wait();
throw new Exception("执行发布错误，错误信息参考上一条消息！");
}
}
else
{
throw new NotSupportedException("只支持windows!");
}
}
}
protected override void HandleLocalMsg(SyncMsg msg) { }
protected override void HandleRemoteMsg(SyncMsg msg)
{
throw new NotImplementedException();
}
}
public class UploadPackedHelper(LocalSyncServer context)
: StateHelpBase(context, SyncProcessStep.UploadAndUnpack)
{
public void UpLoadPackedFile()
{
Context
.LocalPipe.UploadFile(
Context.NotNullSyncConfig.RemoteUrl + "/UploadPacked",
$"{LocalSyncServer.TempRootFile}/{Context.NotNullSyncConfig.Id}/{Context.NotNullSyncConfig.Id}.zip",
(double current) =>
{
Context
.LocalPipe.SendMsg(CreateMsg(current.ToString(), SyncMsgType.Process))
.Wait();
return true;
}
)
.Wait();
Context.LocalPipe.SendMsg(CreateMsg("上传完成！")).Wait();
}
protected override void HandleLocalMsg(SyncMsg msg)
{
throw new NotImplementedException();
}
protected override void HandleRemoteMsg(SyncMsg msg)
{
Context.LocalPipe.SendMsg(msg).Wait();
var h = new FinallyPublishHelper(Context);
Context.StateHelper = h;
}
}
public class FinallyPublishHelper(LocalSyncServer context)
: StateHelpBase(context, SyncProcessStep.Publish)
{
protected override void HandleLocalMsg(SyncMsg msg)
{
throw new NotImplementedException();
}
/// <summary>
/// 由最初始的客户端断开连接，表示发布完成。
/// </summary>
/// <param name="msg"></param>
protected override void HandleRemoteMsg(SyncMsg msg)
{
Context.LocalPipe.SendMsg(msg).Wait();
}
}
// /// <summary>
// /// 0. 发布源验证密码
// /// </summary>

40
Server/RemoteServer/Controllers/RemoteServerController.cs
View file

@ -161,4 +161,44 @@ public class SyncFilesController(RemoteSyncServerFactory factory, SqliteDbContex
return Ok(new { IsSuccess = false, e.Message });
}
}
[HttpPost("/UploadFile")]
public async Task<IActionResult> UploadFile(IFormFile file, [FromQuery] string Id)
{
try
{
if (file == null || file.Length == 0)
{
throw new Exception("文件不存在！");
}
var uploadPath = Path.Combine(RemoteSyncServer.TempRootFile, Id);
if (!Directory.Exists(uploadPath))
Directory.CreateDirectory(uploadPath);
var filePath = Path.Combine(uploadPath, file.FileName);
using (var stream = new FileStream(filePath, FileMode.Create))
{
await file.CopyToAsync(stream);
}
var server = Factory.GetServerById(Id);
if (server == null)
{
throw new Exception("不存在的Id！");
}
else
{
var h = new UnPackAndReleaseHelper(server);
server.StateHelper = h;
h.UnPack();
}
return Ok(new { IsSuccess = true, Message = "File uploaded successfully." });
}
catch (Exception ex)
{
return StatusCode(
500,
new { IsSuccess = false, Message = $"Internal server error: {ex.Message}" }
);
}
}
}

9
Server/RemoteServer/Program.cs
View file

@ -1,4 +1,5 @@
using Microsoft.EntityFrameworkCore;
using Microsoft.OpenApi.Validations;
using RemoteServer;
using RemoteServer.Models;
@ -9,9 +10,10 @@ ConfigurationBuilder configurationBuilder = new();
// Add services to the container.
//添加配置文件路径
RemoteSyncServerFactory.NamePwd = [.. (
builder.Configuration.GetSection("NamePwds").Get<Tuple<string, string>[]>() ?? []
)];
RemoteSyncServerFactory.NamePwd =
[
.. (builder.Configuration.GetSection("NamePwds").Get<Tuple<string, string>[]>() ?? [])
];
RemoteSyncServer.TempRootFile = builder.Configuration["TempDir"] ?? "C:/TempPack";
builder.Services.AddControllers();
builder.Services.AddDbContext<SqliteDbContext>(opions =>
@ -35,5 +37,4 @@ app.UseWebSockets();
app.Urls.Clear();
app.Urls.Add("http://0.0.0.0:6818");
app.MapControllers();
app.Run();

1
Server/RemoteServer/RemoteSyncServer.cs
View file

@ -12,6 +12,7 @@ public class RemoteSyncServer
public StateHelpBase StateHelper;
public Config? SyncConfig;
public List<DirFileConfig> Diff = [];
public Config NotNullSyncConfig
{

5
Server/RemoteServer/RemoteSyncServerFactory.cs
View file

@ -37,4 +37,9 @@ public class RemoteSyncServerFactory
var it = Servers.Where(x => x.Name == name).FirstOrDefault();
return it;
}
public RemoteSyncServer? GetServerById(string Id)
{
return Servers.Where(x => x.NotNullSyncConfig.Id.ToString() == Id).FirstOrDefault();
}
}

111
Server/RemoteServer/StateHelper.cs
View file

@ -21,7 +21,6 @@ public abstract class StateHelpBase(
protected readonly RemoteSyncServer Context = context;
protected readonly SyncProcessStep Step = step;
public SyncMsg CreateErrMsg(string Body)
{
return new SyncMsg(SyncMsgType.Error, Step, Body);
@ -76,6 +75,7 @@ public class DiffFileHelper(RemoteSyncServer context)
protected override void HandleMsg(SyncMsg msg)
{
Context.SyncConfig = JsonSerializer.Deserialize<Config>(msg.Body);
//文件对比
Context.NotNullSyncConfig.DirFileConfigs.ForEach(e =>
{
@ -92,19 +92,116 @@ public class DiffFileHelper(RemoteSyncServer context)
);
nd.ExtractInfo(e.CherryPicks, e.Excludes);
var diff = e.DirInfo.Diff(nd);
e.DirInfo = diff;
e.DirInfo = nd;
Context.Diff.Add(
new DirFileConfig()
{
DirPath = e.DirPath,
Excludes = e.Excludes,
CherryPicks = e.CherryPicks,
DirInfo = diff
}
);
}
});
//将对比结果发送到Local
Context.Pipe.SendMsg(
CreateMsg(JsonSerializer.Serialize(Context.NotNullSyncConfig.DirFileConfigs))
Context.Pipe.SendMsg(CreateMsg(JsonSerializer.Serialize(Context.Diff)));
}
}
public class UnPackAndReleaseHelper(RemoteSyncServer context)
: StateHelpBase(context, SyncProcessStep.UploadAndUnpack)
{
public void UnPack()
{
FileDirOpForUnpack.FirstUnComparess(
Path.Combine(RemoteSyncServer.TempRootFile, Context.NotNullSyncConfig.Id.ToString()),
Context.NotNullSyncConfig.Id.ToString()
);
Context.Pipe.SendMsg(CreateMsg("解压完成！")).Wait();
var h = new FinallyPublishHelper(Context);
Context.StateHelper = h;
h.FinallyPublish();
}
protected override void HandleMsg(SyncMsg msg) { }
}
public class UnPackFilesHelper(RemoteSyncServer context):StateHelpBase(context,SyncProcessStep.UploadAndUnpack)
public class FinallyPublishHelper(RemoteSyncServer context)
: StateHelpBase(context, SyncProcessStep.Publish)
{
protected override void HandleMsg(SyncMsg msg)
public void FinallyPublish()
{
throw new NotImplementedException();
// 发布数据库
if (Context.NotNullSyncConfig.IsDeployDb)
{
if (RuntimeInformation.IsOSPlatform(OSPlatform.Windows))
{
var arguments =
$"SqlPackage /Action:Publish /SourceFile: {RemoteSyncServer.TempRootFile}/{Context.NotNullSyncConfig.Id}/{Context.NotNullSyncConfig.Id}.dacpac "
+ $"/TargetServerName:{Context.NotNullSyncConfig.DstDb.ServerName} /TargetDatabaseName:{Context.NotNullSyncConfig.DstDb.DatebaseName}"
+ $" /TargetUser:{Context.NotNullSyncConfig.DstDb.User} /TargetPassword:{Context.NotNullSyncConfig.DstDb.Password} /TargetTrustServerCertificate:True ";
ProcessStartInfo startInfo =
new()
{
FileName = "cmd.exe", // The command to execute (can be any command line tool)
Arguments = arguments,
// The arguments to pass to the command (e.g., list directory contents)
RedirectStandardOutput = true, // Redirect the standard output to a string
UseShellExecute = false, // Do not use the shell to execute the command
CreateNoWindow = true // Do not create a new window for the command
};
using Process process = new() { StartInfo = startInfo };
// Start the process
process.Start();
// Read the output from the process
string output = process.StandardOutput.ReadToEnd();
// Wait for the process to exit
process.WaitForExit();
if (process.ExitCode == 0)
{
Context.Pipe.SendMsg(CreateMsg("数据库发布成功！")).Wait();
}
else
{
Context.Pipe.SendMsg(CreateErrMsg(output)).Wait();
throw new Exception("执行发布错误，错误信息参考上一条消息！");
}
}
else
{
throw new NotSupportedException("只支持windows!");
}
}
else
{
Context.Pipe.SendMsg(CreateMsg("跳过数据库发布！")).Wait();
}
var DirFileOp = new FileDirOpForUnpack(
Path.Combine(
RemoteSyncServer.TempRootFile,
Context.NotNullSyncConfig.Id.ToString(),
Context.NotNullSyncConfig.Id.ToString()
),
Context.NotNullSyncConfig.RemoteRootPath
);
for (int i = 0; i < Context.NotNullSyncConfig.DirFileConfigs.Count; ++i)
{
#pragma warning disable CS8602 // Dereference of a possibly null reference.
#pragma warning disable CS8604 // Possible null reference argument.
Context
.NotNullSyncConfig.DirFileConfigs[i]
.DirInfo.CombineJustDirFile(DirFileOp, Context.Diff[i].DirInfo);
#pragma warning restore CS8604 // Possible null reference argument.
#pragma warning restore CS8602 // Dereference of a possibly null reference.
}
Context.Pipe.SendMsg(CreateMsg("发布完成！")).Wait();
}
protected override void HandleMsg(SyncMsg msg) { }
}