add(ci): support core build & signing by cert (#166)

* add(ci): github action & signing

* add(ci): github action & signing

* add(ci): github action & signing

* add(ci): github action & signing

* add(ci): github action & signing

* add(ci): github action & signing

* add(ci): github action & signing

* add(ci): github action & signing

* add(ci): github action & signing

* add(ci): github action & signing

* add(ci): github action & signing

* add(ci): github action & signing

* chore: update signature CI

* fix(ci): sign windows binaries with direct CERTUM SHA1

* fix(ci): replace cert-store wait with simplysign readiness check

* feat(ci): refactor certum signing flow for release

* fix(ci): fail certum auth when session not ready

* fix(ci): retry certum auth and fail critical steps

* fix(ci): add cert preflight checks before signtool

* fix(install): update SimplySign Desktop MSI download link to version 9.4.3.90

* fix(ci): mask totp and certificate identifiers in scripts

* fix(ci): remove certificate inventory logs from release

* add(ci): CHANGE beta release

* feat(ci): resolve certum msi url dynamically from download page

* add(ci): github action & signing

* add(ci): github action & signing

* revert(ci): app/app/build.gradle for build

* add(ci): github action & signing

* add(ci): github action & signing

* Update default URLs for binary file downloads

---------

Co-authored-by: Suyunmeng <[email protected]>
Co-authored-by: OpenP2P <[email protected]>

.github/scripts/configure-simplysign-registry.ps1

@@ -0,0 +1,252 @@
+param(
+    [switch]$DebugMode = $false,
+    [switch]$VerifyOnly = $false
+)
+
+# SimplySign Desktop Registry Configuration Script
+# Pre-configures optimal registry settings for automated login dialog display
+
+Write-Host "=== SimplySign Desktop Registry Configuration ==="
+
+if ($DebugMode) {
+    Write-Host "Debug mode enabled - verbose logging active"
+}
+
+# Registry path for SimplySign Desktop settings
+$RegistryPath = "HKCU:\Software\Certum\SimplySign"
+
+# Optimal configuration values for automation
+$OptimalSettings = @{
+    "ShowLoginDialogOnStart" = 1
+    "ShowLoginDialogOnAppRequest" = 1
+    "RememberLastUserName" = 1
+    "Autostart" = 0
+    "UnregisterCertificatesOnDisconnect" = 0
+    "RememberPINinCSP" = 1
+    "ForgetPINinCSPonDisconnect" = 1
+    "LangID" = 9
+}
+
+# Function to check if registry path exists
+function Test-RegistryPath {
+    param([string]$Path)
+    
+    try {
+        $null = Get-Item -Path $Path -ErrorAction Stop
+        return $true
+    } catch {
+        return $false
+    }
+}
+
+# Function to get current registry value
+function Get-RegistryValue {
+    param(
+        [string]$Path,
+        [string]$Name
+    )
+    
+    try {
+        $value = Get-ItemProperty -Path $Path -Name $Name -ErrorAction Stop
+        return $value.$Name
+    } catch {
+        return $null
+    }
+}
+
+# Function to set registry value safely
+function Set-RegistryValue {
+    param(
+        [string]$Path,
+        [string]$Name,
+        [int]$Value
+    )
+    
+    try {
+        Set-ItemProperty -Path $Path -Name $Name -Value $Value -Type DWord -ErrorAction Stop
+        if ($DebugMode) {
+            Write-Host "  Set $Name = $Value"
+        }
+        return $true
+    } catch {
+        Write-Host "  ERROR: Failed to set $Name = $Value - $($_.Exception.Message)"
+        return $false
+    }
+}
+
+# Function to display current settings
+function Show-CurrentSettings {
+    Write-Host "Current SimplySign Desktop registry settings:"
+    Write-Host "============================================="
+    
+    if (-not (Test-RegistryPath $RegistryPath)) {
+        Write-Host "Registry path does not exist: $RegistryPath"
+        return
+    }
+    
+    foreach ($setting in $OptimalSettings.Keys) {
+        $currentValue = Get-RegistryValue -Path $RegistryPath -Name $setting
+        if ($null -eq $currentValue) {
+            Write-Host "  $setting : NOT SET"
+        } else {
+            Write-Host "  $setting : $currentValue"
+        }
+    }
+    Write-Host ""
+}
+
+# Function to create registry structure
+function Initialize-RegistryStructure {
+    Write-Host "Initializing registry structure..."
+    
+    # Create parent keys if they don't exist
+    $ParentPaths = @(
+        "HKCU:\Software\Certum",
+        $RegistryPath
+    )
+    
+    $allCreated = $true
+    foreach ($path in $ParentPaths) {
+        if (-not (Test-RegistryPath $path)) {
+            try {
+                New-Item -Path $path -Force -ErrorAction Stop | Out-Null
+                if ($DebugMode) {
+                    Write-Host "  Created registry path: $path"
+                }
+            } catch {
+                Write-Host "  ERROR: Failed to create registry path: $path - $($_.Exception.Message)"
+                $allCreated = $false
+            }
+        } else {
+            if ($DebugMode) {
+                Write-Host "  Registry path exists: $path"
+            }
+        }
+    }
+    
+    return $allCreated
+}
+
+# Function to apply optimal configuration
+function Set-OptimalConfiguration {
+    Write-Host "Applying optimal configuration for automation..."
+    
+    $successCount = 0
+    $totalSettings = $OptimalSettings.Count
+    
+    foreach ($setting in $OptimalSettings.Keys) {
+        $value = $OptimalSettings[$setting]
+        if (Set-RegistryValue -Path $RegistryPath -Name $setting -Value $value) {
+            $successCount++
+        }
+    }
+    
+    Write-Host "Applied $successCount of $totalSettings settings successfully"
+    return ($successCount -eq $totalSettings)
+}
+
+# Function to verify configuration
+function Test-Configuration {
+    Write-Host "Verifying configuration..."
+    
+    $verificationResults = @{}
+    $allCorrect = $true
+    
+    foreach ($setting in $OptimalSettings.Keys) {
+        $expectedValue = $OptimalSettings[$setting]
+        $actualValue = Get-RegistryValue -Path $RegistryPath -Name $setting
+        
+        $isCorrect = ($actualValue -eq $expectedValue)
+        $verificationResults[$setting] = @{
+            Expected = $expectedValue
+            Actual = $actualValue
+            Correct = $isCorrect
+        }
+        
+        if (-not $isCorrect) {
+            $allCorrect = $false
+        }
+        
+        if ($DebugMode -or -not $isCorrect) {
+            $status = if ($isCorrect) { "OK" } else { "MISMATCH" }
+            Write-Host "  $setting : Expected=$expectedValue, Actual=$actualValue [$status]"
+        }
+    }
+    
+    return $verificationResults, $allCorrect
+}
+
+# Main execution
+try {
+    Write-Host "Starting registry configuration process..."
+    Write-Host ""
+    
+    # Show current state
+    Write-Host "BEFORE CONFIGURATION:"
+    Show-CurrentSettings
+    
+    if ($VerifyOnly) {
+        Write-Host "Verification-only mode - no changes will be made"
+        $verificationResults, $allCorrect = Test-Configuration
+        
+        if ($allCorrect) {
+            Write-Host "SUCCESS: All settings are correctly configured"
+            exit 0
+        } else {
+            Write-Host "CONFIGURATION NEEDED: Some settings require adjustment"
+            exit 1
+        }
+    }
+    
+    # Initialize registry structure
+    if (-not (Initialize-RegistryStructure)) {
+        Write-Host "FATAL ERROR: Failed to initialize registry structure"
+        exit 1
+    }
+    
+    # Apply optimal configuration
+    if (-not (Set-OptimalConfiguration)) {
+        Write-Host "ERROR: Failed to apply complete configuration"
+        exit 1
+    }
+    
+    Write-Host ""
+    Write-Host "AFTER CONFIGURATION:"
+    Show-CurrentSettings
+    
+    # Verify the configuration was applied correctly
+    $verificationResults, $allCorrect = Test-Configuration
+    
+    if ($allCorrect) {
+        Write-Host "SUCCESS: Registry configuration completed successfully"
+        Write-Host ""
+        Write-Host "Key automation settings enabled:"
+        Write-Host "  ShowLoginDialogOnStart = 1 (Login dialog will appear automatically)"
+        Write-Host "  ShowLoginDialogOnAppRequest = 1 (Dialog appears when apps request access)"
+        Write-Host "  RememberLastUserName = 1 (Username persistence for efficiency)"
+        Write-Host ""
+        Write-Host "Next steps:"
+        Write-Host "1. Launch SimplySign Desktop"
+        Write-Host "2. Login dialog should appear automatically"
+        Write-Host "3. Complete authentication process"
+        
+        # Create a status file for the workflow to check
+        "REGISTRY_CONFIGURATION_SUCCESS" | Out-File -FilePath "registry_config_status.log" -Encoding UTF8
+        
+        exit 0
+    } else {
+        Write-Host "ERROR: Configuration verification failed"
+        Write-Host "Some settings were not applied correctly"
+        
+        "REGISTRY_CONFIGURATION_PARTIAL" | Out-File -FilePath "registry_config_status.log" -Encoding UTF8
+        
+        exit 1
+    }
+    
+} catch {
+    Write-Host "FATAL ERROR: Registry configuration failed - $($_.Exception.Message)"
+    
+    "REGISTRY_CONFIGURATION_FAILED" | Out-File -FilePath "registry_config_status.log" -Encoding UTF8
+    
+    exit 1
+}

.github/scripts/connect-simplySign-enhanced.ps1

@@ -0,0 +1,390 @@
+# Connect-SimplySign-Enhanced.ps1
+# Registry-Enhanced TOTP Authentication for SimplySign Desktop
+# Uses registry pre-configuration + TOTP credential injection approach
+
+param(
+    [string]$OtpUri = $env:CERTUM_OTP_URI,
+    [string]$UserId = $env:CERTUM_USERNAME,
+    [string]$ExePath = $env:CERTUM_EXE_PATH,
+    [string]$ExpectedCertificateSHA1 = $env:CERTUM_CERTIFICATE_SHA1
+)
+
+function Normalize-Sha1 {
+    param([string]$InputSha1)
+    if (-not $InputSha1) {
+        return $null
+    }
+    return ($InputSha1 -replace "[^a-fA-F0-9]", "").ToUpperInvariant()
+}
+
+function Find-CertificateByThumbprint {
+    param([string]$Thumbprint)
+
+    if (-not $Thumbprint) {
+        return @()
+    }
+
+    $all = Get-ChildItem -Path "Cert:\CurrentUser\My", "Cert:\LocalMachine\My" -ErrorAction SilentlyContinue
+    # 对证书库中的 Thumbprint 同样做规范化（去除不可见字符、统一大写），避免 BOM 或格式差异导致匹配失败
+    return @($all | Where-Object {
+        $normalizedStoreThumbprint = ($_.Thumbprint -replace "[^a-fA-F0-9]", "").ToUpperInvariant()
+        $normalizedStoreThumbprint -eq $Thumbprint
+    })
+}
+
+# Validate required parameters
+if (-not $OtpUri) {
+    Write-Host "ERROR: CERTUM_OTP_URI environment variable not provided"
+    exit 1
+}
+
+if (-not $UserId) {
+    Write-Host "ERROR: CERTUM_USERNAME environment variable not provided"
+    exit 1
+}
+
+if (-not $ExePath) {
+    $ExePath = "C:\Program Files\Certum\SimplySign Desktop\SimplySignDesktop.exe"
+}
+
+Write-Host "=== REGISTRY-ENHANCED TOTP AUTHENTICATION ==="
+Write-Host "Using registry pre-configuration + credential injection"
+Write-Host "OTP URI provided (length: $($OtpUri.Length))"
+Write-Host "User ID: $UserId"
+Write-Host "Executable: $ExePath"
+Write-Host ""
+
+# Verify SimplySign Desktop exists
+if (-not (Test-Path $ExePath)) {
+    Write-Host "ERROR: SimplySign Desktop not found at: $ExePath"
+    exit 1
+}
+
+# Parse the otpauth:// URI
+$uri = [Uri]$OtpUri
+
+# Parse query parameters (compatible with both PowerShell 5.1 and 7+)
+try {
+    $q = [System.Web.HttpUtility]::ParseQueryString($uri.Query)
+} catch {
+    $q = @{}
+    foreach ($part in $uri.Query.TrimStart('?') -split '&') {
+        $kv = $part -split '=', 2
+        if ($kv.Count -eq 2) { 
+            $q[$kv[0]] = [Uri]::UnescapeDataString($kv[1]) 
+        }
+    }
+}
+
+$Base32 = $q['secret']
+$Digits = if ($q['digits']) { [int]$q['digits'] } else { 6 }
+$Period = if ($q['period']) { [int]$q['period'] } else { 30 }
+$Algorithm = if ($q['algorithm']) { $q['algorithm'].ToUpper() } else { 'SHA256' }
+
+# Validate supported algorithms
+$SupportedAlgorithms = @('SHA1', 'SHA256', 'SHA512')
+if ($Algorithm -notin $SupportedAlgorithms) {
+    Write-Host "ERROR: Unsupported algorithm: $Algorithm. Supported: $($SupportedAlgorithms -join ', ')"
+    exit 1
+}
+
+# TOTP Generator (inline C# implementation)
+Add-Type -Language CSharp @"
+using System;
+using System.Security.Cryptography;
+
+public static class Totp
+{
+    private const string B32 = "ABCDEFGHIJKLMNOPQRSTUVWXYZ234567";
+
+    private static byte[] Base32Decode(string s)
+    {
+        s = s.TrimEnd('=').ToUpperInvariant();
+        int byteCount = s.Length * 5 / 8;
+        byte[] bytes = new byte[byteCount];
+
+        int bitBuffer = 0, bitsLeft = 0, idx = 0;
+        foreach (char c in s)
+        {
+            int val = B32.IndexOf(c);
+            if (val < 0) throw new ArgumentException("Invalid Base32 char: " + c);
+
+            bitBuffer = (bitBuffer << 5) | val;
+            bitsLeft += 5;
+
+            if (bitsLeft >= 8)
+            {
+                bytes[idx++] = (byte)(bitBuffer >> (bitsLeft - 8));
+                bitsLeft -= 8;
+            }
+        }
+        return bytes;
+    }
+
+    private static HMAC GetHmacAlgorithm(string algorithm, byte[] key)
+    {
+        switch (algorithm.ToUpper())
+        {
+            case "SHA1":
+                return new HMACSHA1(key);
+            case "SHA256":
+                return new HMACSHA256(key);
+            case "SHA512":
+                return new HMACSHA512(key);
+            default:
+                throw new ArgumentException("Unsupported algorithm: " + algorithm);
+        }
+    }
+
+    public static string Now(string secret, int digits, int period, string algorithm = "SHA256")
+    {
+        byte[] key = Base32Decode(secret);
+        long counter = DateTimeOffset.UtcNow.ToUnixTimeSeconds() / period;
+
+        byte[] cnt = BitConverter.GetBytes(counter);
+        if (BitConverter.IsLittleEndian) Array.Reverse(cnt);
+
+        byte[] hash;
+        using (var hmac = GetHmacAlgorithm(algorithm, key))
+        {
+            hash = hmac.ComputeHash(cnt);
+        }
+
+        int offset = hash[hash.Length - 1] & 0x0F;
+        int binary =
+            ((hash[offset] & 0x7F) << 24) |
+            ((hash[offset + 1] & 0xFF) << 16) |
+            ((hash[offset + 2] & 0xFF) << 8) |
+            (hash[offset + 3] & 0xFF);
+
+        int otp = binary % (int)Math.Pow(10, digits);
+        return otp.ToString(new string('0', digits));
+    }
+}
+"@
+
+function Get-TotpCode {
+    param([string]$Secret, [int]$Digits = 6, [int]$Period = 30, [string]$Algorithm = 'SHA256')
+    [Totp]::Now($Secret, $Digits, $Period, $Algorithm)
+}
+
+# Add Win32 API for force foreground window
+Add-Type @"
+using System;
+using System.Runtime.InteropServices;
+
+public static class Win32 {
+    [DllImport("user32.dll")]
+    public static extern bool SetForegroundWindow(IntPtr hWnd);
+
+    [DllImport("user32.dll")]
+    public static extern bool ShowWindow(IntPtr hWnd, int nCmdShow);
+
+    [DllImport("user32.dll")]
+    public static extern IntPtr FindWindow(string lpClassName, string lpWindowName);
+
+    [DllImport("user32.dll")]
+    public static extern bool AllowSetForegroundWindow(int dwProcessId);
+
+    public const int SW_RESTORE = 9;
+    public const int SW_SHOW = 5;
+}
+"@
+
+# 预先验证证书 SHA1
+$normalizedExpectedSha1 = Normalize-Sha1 -InputSha1 $ExpectedCertificateSHA1
+if ($normalizedExpectedSha1) {
+    if ($normalizedExpectedSha1.Length -ne 40) {
+        Write-Host "ERROR: CERTUM_CERTIFICATE_SHA1 is invalid after normalization"
+        Write-Host "Raw length: $($ExpectedCertificateSHA1.Length), normalized length: $($normalizedExpectedSha1.Length)"
+        exit 1
+    }
+}
+
+# === 认证重试循环（最多 10 次） ===
+$maxAttempts = 10
+$authSuccess = $false
+
+for ($attempt = 1; $attempt -le $maxAttempts; $attempt++) {
+    Write-Host ""
+    Write-Host "=========================================="
+    Write-Host "=== AUTHENTICATION ATTEMPT $attempt / $maxAttempts ==="
+    Write-Host "=========================================="
+    Write-Host ""
+
+    # 每次重试都重新生成 TOTP（确保验证码有效）
+    $otp = Get-TotpCode -Secret $Base32 -Digits $Digits -Period $Period -Algorithm $Algorithm
+    Write-Host "Generated TOTP code successfully (masked) using $Algorithm algorithm"
+    Write-Host ""
+
+    # 终止之前可能残留的 SimplySign Desktop 进程
+    Get-Process -Name "SimplySignDesktop" -ErrorAction SilentlyContinue | Stop-Process -Force -ErrorAction SilentlyContinue
+    Start-Sleep -Seconds 2
+
+    # 启动 SimplySign Desktop
+    Write-Host "Launching SimplySign Desktop..."
+    Write-Host "Registry pre-configuration should auto-open login dialog"
+    $proc = Start-Process -FilePath $ExePath -PassThru
+    Write-Host "Process started with ID: $($proc.Id)"
+    Write-Host ""
+
+    # 等待应用初始化
+    Write-Host "Waiting for SimplySign Desktop to initialize..."
+    Start-Sleep -Seconds 10
+
+    # Allow our process to set foreground window
+    [Win32]::AllowSetForegroundWindow($proc.Id) | Out-Null
+
+    # Create WScript.Shell for window interaction
+    $wshell = New-Object -ComObject WScript.Shell
+
+    # 尝试聚焦 SimplySign Desktop 窗口
+    Write-Host "Attempting to focus SimplySign Desktop window..."
+    $focused = $false
+
+    # Method 1: Use Win32 API to find and activate window
+    $mainWindowHandle = $proc.MainWindowHandle
+    if ($mainWindowHandle -ne $null -and $mainWindowHandle -ne [IntPtr]::Zero) {
+        [Win32]::ShowWindow($mainWindowHandle, [Win32]::SW_RESTORE) | Out-Null
+        [Win32]::SetForegroundWindow($mainWindowHandle) | Out-Null
+        $focused = $true
+        Write-Host "Focused via MainWindowHandle"
+    } else {
+        Write-Host "MainWindowHandle not available yet, will try other methods..."
+    }
+
+    # Method 2: Find window by title
+    if (-not $focused) {
+        $hwnd = [Win32]::FindWindow($null, "SimplySign Desktop")
+        if ($hwnd -ne [IntPtr]::Zero) {
+            [Win32]::ShowWindow($hwnd, [Win32]::SW_RESTORE) | Out-Null
+            [Win32]::SetForegroundWindow($hwnd) | Out-Null
+            $focused = $true
+            Write-Host "Focused via FindWindow"
+        }
+    }
+
+    # Method 3: AppActivate with extended retries
+    for ($i = 0; (-not $focused) -and ($i -lt 20); $i++) {
+        Start-Sleep -Milliseconds 1000
+
+        # Refresh process handle
+        $proc.Refresh()
+        $mainWindowHandle = $proc.MainWindowHandle
+        if ($mainWindowHandle -ne [IntPtr]::Zero) {
+            [Win32]::ShowWindow($mainWindowHandle, [Win32]::SW_RESTORE) | Out-Null
+            [Win32]::SetForegroundWindow($mainWindowHandle) | Out-Null
+            $focused = $true
+            Write-Host "Focused via MainWindowHandle (attempt $($i + 1))"
+            break
+        }
+
+        $focused = $wshell.AppActivate($proc.Id)
+        if (-not $focused) {
+            $focused = $wshell.AppActivate('SimplySign Desktop')
+        }
+        if (-not $focused) {
+            $focused = $wshell.AppActivate('SimplySign')
+        }
+        Write-Host "Focus attempt $($i + 1): $focused"
+    }
+
+    if (-not $focused) {
+        Write-Host "WARNING: Could not bring SimplySign Desktop to foreground via window handle"
+        Write-Host "SimplySign Desktop may be running as a background/tray process - proceeding with credential injection anyway"
+    }
+
+    Write-Host ""
+
+    # Small delay to ensure window is ready for input
+    Start-Sleep -Milliseconds 400
+
+    # 注入凭据: Username + TAB + TOTP + ENTER
+    Write-Host "Injecting credentials into login dialog..."
+    Write-Host "Sending: Username -> TAB -> TOTP -> ENTER"
+
+    $wshell.SendKeys($UserId)
+    Start-Sleep -Milliseconds 200
+    $wshell.SendKeys("{TAB}")
+    Start-Sleep -Milliseconds 200
+    $wshell.SendKeys($otp)
+    Start-Sleep -Milliseconds 200
+    $wshell.SendKeys("{ENTER}")
+
+    Write-Host "Credentials injected successfully"
+    Write-Host ""
+
+    # 等待认证处理
+    Write-Host "Waiting for authentication to complete..."
+    Start-Sleep -Seconds 5
+
+    # 验证证书是否可用
+    if ($normalizedExpectedSha1) {
+        Write-Host "Validating certificate availability for expected signing certificate"
+        $ready = $false
+        $withPrivateKey = $false
+
+        for ($i = 0; $i -lt 15; $i++) {
+            $matched = Find-CertificateByThumbprint -Thumbprint $normalizedExpectedSha1
+            if ($matched.Count -gt 0) {
+                $ready = $true
+                $withPrivateKey = ($matched | Where-Object { $_.HasPrivateKey }).Count -gt 0
+                if ($withPrivateKey) {
+                    Write-Host "Certificate is available and has private key"
+                    break
+                }
+            }
+            Start-Sleep -Seconds 2
+        }
+
+        if ($ready -and $withPrivateKey) {
+            $authSuccess = $true
+            Write-Host "SUCCESS: Authentication verified - certificate with private key is available"
+            break
+        }
+
+        # 认证失败，准备重试
+        if (-not $ready) {
+            Write-Host "WARNING: Target certificate was not found after attempt $attempt"
+        } elseif (-not $withPrivateKey) {
+            Write-Host "WARNING: Target certificate found but no private key available after attempt $attempt"
+        }
+    } else {
+        # 没有指定证书 SHA1，无法验证，假设成功
+        $authSuccess = $true
+        break
+    }
+
+    # 如果不是最后一次尝试，等待后重试
+    if ($attempt -lt $maxAttempts) {
+        Write-Host "Authentication attempt $attempt failed, will retry in 5 seconds..."
+        # 终止当前 SimplySign Desktop 进程
+        $stillRunning = Get-Process -Id $proc.Id -ErrorAction SilentlyContinue
+        if ($stillRunning) {
+            Stop-Process -Id $proc.Id -Force -ErrorAction SilentlyContinue
+        }
+        Start-Sleep -Seconds 5
+    }
+}
+
+if (-not $authSuccess) {
+    Write-Host ""
+    Write-Host "ERROR: Authentication failed after $maxAttempts attempts"
+    Write-Host "All TOTP injection attempts were unsuccessful"
+    exit 1
+}
+
+# Verify SimplySign Desktop is still running
+$stillRunning = Get-Process -Id $proc.Id -ErrorAction SilentlyContinue
+if ($stillRunning) {
+    Write-Host "SUCCESS: SimplySign Desktop is running"
+    Write-Host "Authentication should be complete"
+    Write-Host "Cloud certificate should now be available"
+} else {
+    Write-Host "WARNING: SimplySign Desktop process has exited"
+    Write-Host "This may indicate authentication failure"
+}
+
+Write-Host ""
+Write-Host "=== TOTP AUTHENTICATION COMPLETE ==="
+Write-Host "Registry pre-configuration + credential injection finished"

.github/scripts/install-simplysign.sh

@@ -0,0 +1,138 @@
+#!/bin/bash
+
+# Install SimplySign Desktop - Clean MSI Installation
+
+set -euo pipefail
+
+echo "=== INSTALLING SIMPLYSIGN DESKTOP ==="
+echo "Using proven installation method from successful testing..."
+
+# Download SimplySign Desktop MSI
+CERTUM_INSTALLER="SimplySignDesktop.msi"
+CERTUM_DOWNLOAD_PAGE="https://pomoc.certum.pl/pl/oprogramowanie/procertum-smartsign/"
+FALLBACK_MSI_URL="https://files.certum.eu/software/SimplySignDesktop/Windows/9.4.3.90/SimplySignDesktop-9.4.3.90-64-bit-pl.msi"
+echo "Downloading SimplySign Desktop MSI..."
+
+# Resolve the latest 64-bit MSI URL from Certum software page to avoid hardcoded version expiry.
+PAGE_CONTENT="$(curl -L "$CERTUM_DOWNLOAD_PAGE" --fail --max-time 60 || true)"
+
+MSI_CANDIDATES="$(printf '%s' "$PAGE_CONTENT" | grep -oE 'https://(www\.)?files\.certum\.eu/software/SimplySignDesktop/Windows/[0-9]+\.[0-9]+\.[0-9]+\.[0-9]+/SimplySignDesktop-[0-9]+\.[0-9]+\.[0-9]+\.[0-9]+-64-bit[^"[:space:]]*\.msi' | sort -u || true)"
+
+DOWNLOAD_URL=""
+if [ -n "$MSI_CANDIDATES" ]; then
+  LATEST_VERSION="$(printf '%s\n' "$MSI_CANDIDATES" | sed -E 's#^.*/Windows/([0-9.]+)/.*$#\1#' | sort -V | tail -n1)"
+  LATEST_VERSION_URLS="$(printf '%s\n' "$MSI_CANDIDATES" | grep "/Windows/${LATEST_VERSION}/" || true)"
+  DOWNLOAD_URL="$(printf '%s\n' "$LATEST_VERSION_URLS" | grep -- '-64-bit-pl\.msi$' | head -n1 || true)"
+
+  if [ -z "$DOWNLOAD_URL" ]; then
+    DOWNLOAD_URL="$(printf '%s\n' "$LATEST_VERSION_URLS" | head -n1)"
+  fi
+fi
+
+if [ -z "$DOWNLOAD_URL" ]; then
+  echo "WARNING: Could not resolve latest MSI URL from Certum page, using fallback URL"
+  DOWNLOAD_URL="$FALLBACK_MSI_URL"
+fi
+
+RESOLVED_VERSION="$(printf '%s' "$DOWNLOAD_URL" | sed -E 's#^.*/Windows/([0-9.]+)/.*$#\1#')"
+echo "Resolved SimplySign Desktop MSI version: $RESOLVED_VERSION"
+
+if curl -L "$DOWNLOAD_URL" -o "$CERTUM_INSTALLER" --fail --max-time 60; then
+  echo "✅ Downloaded SimplySign Desktop MSI ($(ls -lh "$CERTUM_INSTALLER" | awk '{print $5}'))"
+else
+  echo "❌ Failed to download SimplySign Desktop"
+  exit 1
+fi
+
+# Install with proven method (matching successful test)
+echo "Installing SimplySign Desktop..."
+echo "Full command: msiexec /i \"$CERTUM_INSTALLER\" /quiet /norestart /l*v install.log ALLUSERS=1 REBOOT=ReallySuppress"
+
+# Check for administrative privileges (like the successful test)
+ADMIN_RIGHTS=false
+if powershell -Command "([Security.Principal.WindowsPrincipal][Security.Principal.WindowsIdentity]::GetCurrent()).IsInRole([Security.Principal.WindowsBuiltInRole]::Administrator)" 2>/dev/null; then
+  echo "✅ Running with administrative privileges"
+  ADMIN_RIGHTS=true
+else
+  echo "⚠️ No explicit administrative privileges detected"
+fi
+
+# Use the exact method that worked: PowerShell with admin privileges
+if [ "$ADMIN_RIGHTS" = true ]; then
+  echo "Running MSI installation with administrator privileges..."
+  powershell -Command "Start-Process -FilePath 'msiexec.exe' -ArgumentList '/i', '\"$CERTUM_INSTALLER\"', '/quiet', '/norestart', '/l*v', 'install.log', 'ALLUSERS=1', 'REBOOT=ReallySuppress' -Wait -NoNewWindow -PassThru" &
+  INSTALL_PID=$!
+else
+  echo "Running MSI installation without explicit admin elevation..."
+  timeout 300 msiexec /i "$CERTUM_INSTALLER" /quiet /norestart /l*v install.log ALLUSERS=1 REBOOT=ReallySuppress &
+  INSTALL_PID=$!
+fi
+
+# Monitor with the same logic as successful test
+echo "Monitoring installation progress..."
+INSTALL_START_TIME=$(date +%s)
+sleep 10
+
+# Check if msiexec process is actually running (like successful test)
+if kill -0 $INSTALL_PID 2>/dev/null; then
+  echo "MSI installation process is running (PID: $INSTALL_PID)"
+  
+  # Monitor for up to 3 minutes with status updates
+  for i in {1..18}; do
+    sleep 10
+    CURRENT_TIME=$(date +%s)
+    ELAPSED=$((CURRENT_TIME - INSTALL_START_TIME))
+    
+    if kill -0 $INSTALL_PID 2>/dev/null; then
+      echo "Installation still running after ${ELAPSED} seconds..."
+      
+      # Check log file growth
+      if [ -f "install.log" ]; then
+        LOG_SIZE=$(stat -c%s "install.log" 2>/dev/null || stat -f%z "install.log" 2>/dev/null || echo 0)
+        echo "  Log file size: $LOG_SIZE bytes"
+      fi
+    else
+      echo "MSI installation completed after ${ELAPSED} seconds"
+      break
+    fi
+  done
+  
+  # Final wait if still running
+  if kill -0 $INSTALL_PID 2>/dev/null; then
+    echo "Installation taking longer, waiting for completion..."
+    wait $INSTALL_PID 2>/dev/null || echo "Installation process ended"
+  fi
+else
+  echo "MSI installation process ended quickly"
+fi
+
+# Quick success check using proven patterns
+INSTALLATION_SUCCESSFUL=false
+if [ -f "install.log" ]; then
+  if grep -qi "Installation.*operation.*completed.*successfully\|Installation.*success.*or.*error.*status.*0\|MainEngineThread.*is.*returning.*0\|Windows.*Installer.*installed.*the.*product" install.log 2>/dev/null; then
+    echo "✅ Installation successful (confirmed by log patterns)"
+    INSTALLATION_SUCCESSFUL=true
+  fi
+fi
+
+# Verify installation directory
+INSTALL_PATH="/c/Program Files/Certum/SimplySign Desktop"
+if [ -d "$INSTALL_PATH" ]; then
+  echo "✅ SimplySign Desktop installed successfully"
+  echo "✅ Virtual card emulation now active for code signing"
+  INSTALLATION_SUCCESSFUL=true
+  
+  # Set output for GitHub Actions
+  if [ -n "${GITHUB_OUTPUT:-}" ]; then
+    echo "SIMPLYSIGN_PATH=$INSTALL_PATH" >> "$GITHUB_OUTPUT"
+  fi
+fi
+
+if [ "$INSTALLATION_SUCCESSFUL" = false ]; then
+  echo "❌ Installation verification failed"
+  echo "Last 10 lines of install log:"
+  tail -10 install.log 2>/dev/null || echo "No install log available"
+  exit 1
+fi
+
+echo "🎉 SimplySign Desktop installation completed successfully!"

.github/scripts/sign-windows.ps1

@@ -0,0 +1,182 @@
+# Sign-Windows.ps1
+# Signs Windows EasyTier executables and libraries with a Certum SimplySign cloud certificate.
+
+param(
+    [string]$TargetDirectory = "sign_binaries",
+    [string]$CertificateSHA1 = $env:CERTUM_CERTIFICATE_SHA1,
+    [string]$TimestampServer = "http://time.certum.pl"
+)
+
+function Get-LatestSignToolPath {
+    $windowsKitsBin = Join-Path ${env:ProgramFiles(x86)} "Windows Kits\10\bin"
+    if (Test-Path $windowsKitsBin) {
+        $candidate = (
+            Get-ChildItem -Path $windowsKitsBin -Recurse -File -Filter "signtool.exe" -ErrorAction SilentlyContinue |
+                Where-Object { $_.FullName -match "\\x64\\signtool\.exe$" } |
+                ForEach-Object {
+                    $version = [version]"0.0"
+                    if ($_.FullName -match "\\bin\\([^\\]+)\\x64\\signtool\.exe$") {
+                        try {
+                            $version = [version]$matches[1]
+                        } catch {
+                            $version = [version]"0.0"
+                        }
+                    }
+                    [PSCustomObject]@{
+                        Path = $_.FullName
+                        Version = $version
+                    }
+                } |
+                Sort-Object -Property Version -Descending |
+                Select-Object -First 1
+        )
+
+        if ($candidate) {
+            return $candidate.Path
+        }
+    }
+
+    $cmd = Get-Command "signtool.exe" -ErrorAction SilentlyContinue
+    if ($cmd) {
+        return $cmd.Source
+    }
+
+    return $null
+}
+
+function Find-TargetCertificate {
+    param([string]$Thumbprint)
+
+    $all = Get-ChildItem -Path "Cert:\CurrentUser\My", "Cert:\LocalMachine\My" -ErrorAction SilentlyContinue
+    # 对证书库中的 Thumbprint 同样做规范化（去除不可见字符、统一大写），避免 BOM 或格式差异导致匹配失败
+    return @($all | Where-Object {
+        $normalizedStoreThumprint = ($_.Thumbprint -replace "[^a-fA-F0-9]", "").ToUpperInvariant()
+        $normalizedStoreThumprint -eq $Thumbprint
+    })
+}
+
+function Show-PrivateKeyCertificateHints {
+    $candidates = Get-ChildItem -Path "Cert:\CurrentUser\My", "Cert:\LocalMachine\My" -ErrorAction SilentlyContinue |
+        Where-Object { $_.HasPrivateKey }
+
+    if (($null -eq $candidates) -or ($candidates.Count -eq 0)) {
+        Write-Host "No certificates with private keys were found in Personal stores"
+        return
+    }
+
+    Write-Host "Certificates with private keys are present in Personal stores, but details are hidden for security"
+}
+
+Write-Host "=== WINDOWS BINARY SIGNING (CERTUM SIMPLYSIGN) ==="
+Write-Host "Target directory: $TargetDirectory"
+
+if (-not (Test-Path $TargetDirectory)) {
+    Write-Host "ERROR: Target directory not found: $TargetDirectory"
+    exit 1
+}
+
+if (-not $CertificateSHA1) {
+    Write-Host "ERROR: CERTUM_CERTIFICATE_SHA1 environment variable not provided"
+    exit 1
+}
+
+$normalizedSha1 = ($CertificateSHA1 -replace "[^a-fA-F0-9]", "").ToUpperInvariant()
+if ($normalizedSha1.Length -ne 40) {
+    Write-Host "ERROR: CERTUM_CERTIFICATE_SHA1 is invalid after normalization"
+    Write-Host "Raw length: $($CertificateSHA1.Length), normalized length: $($normalizedSha1.Length)"
+    exit 1
+}
+
+Write-Host "Expected signing certificate thumbprint has been received (masked)"
+
+$targetCerts = Find-TargetCertificate -Thumbprint $normalizedSha1
+if (($null -eq $targetCerts) -or ($targetCerts.Count -eq 0)) {
+    Write-Host "ERROR: Target certificate not found in Cert:\CurrentUser\My or Cert:\LocalMachine\My"
+    Write-Host "Authentication likely failed or CERTUM_CERTIFICATE_SHA1 is incorrect"
+    Show-PrivateKeyCertificateHints
+    exit 1
+}
+
+$targetWithPrivateKey = @($targetCerts | Where-Object { $_.HasPrivateKey })
+if (($null -eq $targetWithPrivateKey) -or ($targetWithPrivateKey.Count -eq 0)) {
+    Write-Host "ERROR: Target certificate exists but has no available private key"
+    Write-Host "Signing cannot continue without private key access"
+    Show-PrivateKeyCertificateHints
+    exit 1
+}
+
+Write-Host "Locating signtool..."
+$signTool = Get-LatestSignToolPath
+if (-not $signTool) {
+    Write-Host "ERROR: signtool.exe not found"
+    exit 1
+}
+
+Write-Host "Found signtool: $signTool"
+
+Write-Host "Scanning for Windows binaries to sign (.exe, .dll)..."
+$filesToSign = Get-ChildItem -Path $TargetDirectory -Recurse -File |
+    Where-Object { $_.Extension -iin @(".exe", ".dll") }
+
+if (($null -eq $filesToSign) -or ($filesToSign.Count -eq 0)) {
+    Write-Host "WARNING: No signable files (.exe, .dll) found to sign"
+    exit 0
+}
+
+Write-Host "Found $($filesToSign.Count) files to sign"
+$signedCount = 0
+$failedCount = 0
+
+foreach ($file in $filesToSign) {
+    Write-Host "=== Signing: $($file.Name) ==="
+    Write-Host "Path: $($file.FullName)"
+
+    $attempts = @(
+        @{ Name = "SHA1 thumbprint + /td SHA256"; Args = @("sign", "/sha1", $normalizedSha1, "/tr", $TimestampServer, "/td", "SHA256", "/fd", "SHA256", "/v", $file.FullName) },
+        @{ Name = "SHA1 thumbprint in CurrentUser\\My"; Args = @("sign", "/sha1", $normalizedSha1, "/s", "My", "/tr", $TimestampServer, "/td", "SHA256", "/fd", "SHA256", "/v", $file.FullName) },
+        @{ Name = "SHA1 thumbprint in LocalMachine\\My"; Args = @("sign", "/sha1", $normalizedSha1, "/sm", "/s", "My", "/tr", $TimestampServer, "/td", "SHA256", "/fd", "SHA256", "/v", $file.FullName) },
+        @{ Name = "Auto-select cert (fallback)"; Args = @("sign", "/a", "/tr", $TimestampServer, "/td", "SHA256", "/fd", "SHA256", "/v", $file.FullName) }
+    )
+
+    $signed = $false
+    foreach ($attempt in $attempts) {
+        Write-Host "Attempt: $($attempt.Name)"
+        $signOutput = & $signTool @($attempt.Args) 2>&1
+        if ($LASTEXITCODE -eq 0) {
+            Write-Host "SUCCESS: $($attempt.Name)"
+            $signed = $true
+            break
+        }
+
+        Write-Host "FAILED: $($attempt.Name)"
+        Write-Host "signtool returned a non-zero exit code; detailed output is hidden for security"
+    }
+
+    if ($signed) {
+        $signedCount++
+        $verifyOutput = & $signTool verify /pa /v $file.FullName 2>&1
+        if ($LASTEXITCODE -eq 0) {
+            Write-Host "VERIFIED: Signature verification successful"
+        } else {
+            Write-Host "WARNING: Signature verification failed"
+            Write-Host "Detailed verification output is hidden for security"
+        }
+    } else {
+        $failedCount++
+    }
+
+    Write-Host ""
+}
+
+Write-Host "=== SIGNING SUMMARY ==="
+Write-Host "Total files: $($filesToSign.Count)"
+Write-Host "Successfully signed: $signedCount"
+Write-Host "Failed to sign: $failedCount"
+
+if ($failedCount -eq 0) {
+    Write-Host "ALL WINDOWS BINARIES SIGNED SUCCESSFULLY"
+    exit 0
+}
+
+Write-Host "SOME WINDOWS BINARIES FAILED TO SIGN"
+exit 1

.github/workflows/release.yml

@@ -0,0 +1,510 @@
+name: OpenP2P Release
+
+on:
+  #push:
+  #  branches: [master]
+  workflow_dispatch:
+    inputs:
+      release_type:
+        description: '发布类型 (beta: 测试版, stable: 正式版)'
+        type: choice
+        options:
+          - beta
+          - stable
+        default: beta
+        required: true
+      version:
+        description: '正式版版本号 (e.g. v3.25.11)，beta发布时可留空'
+        type: string
+        required: false
+
+permissions:
+  contents: write
+
+env:
+  BINARY_NAME: openp2p
+
+jobs:
+  build:
+    name: Build ${{ matrix.os }}-${{ matrix.arch }}
+    runs-on: ${{ matrix.runner }}
+    strategy:
+      fail-fast: false
+      matrix:
+        include:
+          # Windows
+          - { os: windows, arch: amd64, runner: ubuntu-latest, goos: windows, goarch: amd64, ext: .exe }
+          - { os: windows, arch: arm64, runner: ubuntu-latest, goos: windows, goarch: arm64, ext: .exe }
+          - { os: windows, arch: i386, runner: ubuntu-latest, goos: windows, goarch: 386, ext: .exe }
+          # Linux
+          - { os: linux, arch: amd64, runner: ubuntu-latest, goos: linux, goarch: amd64, ext: '' }
+          - { os: linux, arch: arm64, runner: ubuntu-latest, goos: linux, goarch: arm64, ext: '' }
+          - { os: linux, arch: i386, runner: ubuntu-latest, goos: linux, goarch: 386, ext: '' }
+          - { os: linux, arch: mips, runner: ubuntu-latest, goos: linux, goarch: mips, ext: '' }
+          - { os: linux, arch: mips64, runner: ubuntu-latest, goos: linux, goarch: mips64, ext: '' }
+          # Darwin
+          - { os: darwin, arch: amd64, runner: ubuntu-latest, goos: darwin, goarch: amd64, ext: '' }
+          - { os: darwin, arch: arm64, runner: ubuntu-latest, goos: darwin, goarch: arm64, ext: '' }
+
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v6
+
+      - name: Setup Go
+        uses: actions/setup-go@v5
+        with:
+          go-version: '1.20'
+
+      - name: Build
+        env:
+          GOOS: ${{ matrix.goos }}
+          GOARCH: ${{ matrix.goarch }}
+          CGO_ENABLED: '0'
+          GOPROXY: https://goproxy.io,direct
+        run: |
+          OUTPUT_NAME="${{ env.BINARY_NAME }}-${{ matrix.os }}-${{ matrix.arch }}${{ matrix.ext }}"
+          go build -trimpath -ldflags="-s -w" -o "$OUTPUT_NAME" cmd/openp2p.go
+          echo "OUTPUT_NAME=$OUTPUT_NAME" >> $GITHUB_ENV
+
+      - name: Upload artifact
+        uses: actions/upload-artifact@v4
+        with:
+          name: ${{ env.BINARY_NAME }}-${{ matrix.os }}-${{ matrix.arch }}
+          path: ${{ env.OUTPUT_NAME }}
+          retention-days: 7
+
+  # build-android:
+  #   name: Build Android APK
+  #   runs-on: ubuntu-latest
+  #   steps:
+  #     - name: Checkout
+  #       uses: actions/checkout@v6
+  #
+  #     - name: Setup Go
+  #       uses: actions/setup-go@v5
+  #       with:
+  #         go-version: '1.23'
+  #
+  #     - name: Setup JDK
+  #       uses: actions/setup-java@v4
+  #       with:
+  #         java-version: '17'
+  #         distribution: 'temurin'
+  #
+  #     - name: Setup Android SDK & NDK
+  #       uses: android-actions/setup-android@v3
+  #       with:
+  #         packages: 'build-tools;30.0.3 ndk;21.4.7075529 platform-tools platforms;android-31'
+  #
+  #     - name: Setup Android Environment
+  #       run: |
+  #         echo "$ANDROID_HOME/ndk/21.4.7075529" >> $GITHUB_PATH
+  #         echo "ANDROID_NDK_HOME=$ANDROID_HOME/ndk/21.4.7075529" >> $GITHUB_ENV
+  #
+  #     - name: Build Go mobile library (gomobile bind)
+  #       env:
+  #         GOPROXY: https://goproxy.io,direct
+  #       run: |
+  #         # Install gomobile and gobind at the same pinned commit to avoid @latest resolution
+  #         go install golang.org/x/mobile/cmd/gomobile@7c4916698cc93475ebfea76748ee0faba2deb2a5
+  #         go install golang.org/x/mobile/cmd/gobind@7c4916698cc93475ebfea76748ee0faba2deb2a5
+  #         gomobile init
+  #         go get -v golang.org/x/mobile/bind@7c4916698cc93475ebfea76748ee0faba2deb2a5
+  #
+  #         cd core
+  #         gomobile bind -target android -v -androidapi 16
+  #
+  #         # Copy artifacts to app libs
+  #         mkdir -p ../app/app/libs
+  #         cp openp2p.aar openp2p-sources.jar ../app/app/libs/
+  #         echo "Go mobile library built and copied to app/app/libs/"
+  #         ls -la ../app/app/libs/
+  #
+  #     - name: Build unsigned APK
+  #       working-directory: app
+  #       run: |
+  #         chmod +x gradlew
+  #         ./gradlew assembleRelease
+  #
+  #         # Find the built APK
+  #         APK_PATH=$(find . -name "*.apk" -path "*/release/*" | head -1)
+  #         if [ -z "$APK_PATH" ]; then
+  #           APK_PATH=$(find . -name "*.apk" | head -1)
+  #         fi
+  #
+  #         if [ -n "$APK_PATH" ]; then
+  #           cp "$APK_PATH" ../openp2p-android.apk
+  #           echo "APK built: $APK_PATH"
+  #         else
+  #           echo "ERROR: No APK found"
+  #           exit 1
+  #         fi
+  #
+  #     - name: Upload APK artifact
+  #       uses: actions/upload-artifact@v4
+  #       with:
+  #         name: openp2p-android-apk
+  #         path: openp2p-android.apk
+  #         retention-days: 7
+
+  sign:
+    name: Sign Artifacts (Certum SimplySign)
+    needs: [build]
+    runs-on: windows-latest
+    continue-on-error: true
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v6
+
+      - name: Download Windows artifacts
+        uses: actions/download-artifact@v4
+        with:
+          pattern: openp2p-windows-*
+          path: sign_binaries/windows
+          merge-multiple: true
+
+      # - name: Download Android APK
+      #   uses: actions/download-artifact@v4
+      #   with:
+      #     name: openp2p-android-apk
+      #     path: sign_binaries/android
+
+      # - name: Setup JDK (for jarsigner)
+      #   uses: actions/setup-java@v4
+      #   with:
+      #     java-version: '17'
+      #     distribution: 'temurin'
+
+      - name: Setup Certum Code Signing (Windows)
+        shell: bash
+        run: |
+          echo "=== SETTING UP CERTUM CODE SIGNING FOR WINDOWS ==="
+          echo "Installing SimplySign Desktop and configuring for automatic authentication"
+
+          chmod +x ./.github/scripts/install-simplysign.sh
+          ./.github/scripts/install-simplysign.sh
+
+          echo "Configuring registry for automatic login dialog..."
+          powershell -ExecutionPolicy Bypass -File "./.github/scripts/configure-simplysign-registry.ps1"
+
+          echo "Certum signing environment ready"
+
+      - name: Authenticate Certum (Windows)
+        shell: bash
+        env:
+          CERTUM_OTP_URI: ${{ secrets.CERTUM_OTP_URI }}
+          CERTUM_USERNAME: ${{ secrets.CERTUM_USERNAME }}
+          CERTUM_CERTIFICATE_SHA1: ${{ secrets.CERTUM_CERTIFICATE_SHA1 }}
+          CERTUM_EXE_PATH: ${{ secrets.CERTUM_EXE_PATH }}
+        run: |
+          echo "=== CERTUM AUTHENTICATION ==="
+          echo "Authenticating with Certum cloud certificate using TOTP"
+
+          for attempt in 1 2 3; do
+            echo "Authentication attempt ${attempt}/3"
+            if powershell -ExecutionPolicy Bypass -File "./.github/scripts/connect-simplySign-enhanced.ps1"; then
+              echo "Authentication completed"
+              exit 0
+            fi
+
+            if [ "$attempt" -lt 3 ]; then
+              echo "Authentication attempt failed, retrying in 10 seconds..."
+              sleep 10
+            fi
+          done
+
+          echo "ERROR: Certum authentication failed after 3 attempts"
+          exit 1
+
+      - name: Verify Certificate and Sign Windows Binaries
+        shell: bash
+        env:
+          CERTUM_CERTIFICATE_SHA1: ${{ secrets.CERTUM_CERTIFICATE_SHA1 }}
+        run: |
+          echo "=== CERTIFICATE VERIFICATION AND WINDOWS BINARY SIGNING ==="
+          echo "Allowing connection to stabilize..."
+          sleep 10
+
+          echo "Comprehensive certificate availability check..."
+          echo ""
+
+          echo "Skipping certificate store dump to avoid exposing certificate metadata in logs"
+
+          echo ""
+          echo "=== PKCS#11 Library Check ==="
+          if [ -f "/c/Windows/System32/SimplySignPKCS.dll" ]; then
+            echo "PKCS#11 library present: /c/Windows/System32/SimplySignPKCS.dll"
+            ls -la "/c/Windows/System32/SimplySignPKCS.dll"
+          else
+            echo "PKCS#11 library not found"
+          fi
+
+          echo ""
+          echo "=== SimplySign Desktop Status ==="
+          powershell -Command "
+            Write-Host 'SimplySign Desktop process status:'
+            Get-Process -Name '*SimplySign*' -ErrorAction SilentlyContinue |
+              Select-Object Name, Id, MainWindowTitle, Responding |
+              Format-Table -AutoSize
+          "
+
+          echo ""
+          echo "Certificate debugging completed - proceeding to signing..."
+          echo ""
+
+          powershell -ExecutionPolicy Bypass -File "./.github/scripts/sign-windows.ps1" -TargetDirectory "sign_binaries/windows"
+
+          echo "Windows binary signing completed"
+
+      # - name: Sign Android APK (jarsigner + PKCS#11)
+      #   shell: pwsh
+      #   run: |
+      #     Write-Host "=== SIGNING ANDROID APK WITH CERTUM CLOUD CERTIFICATE (PKCS#11) ==="
+      #
+      #     # Create PKCS#11 config file for Certum SimplySign
+      #     $pkcs11Config = @"
+      #     name = CertumSimplySign
+      #     library = C:\Program Files\Certum\SimplySign Desktop\cryptoCertum3PKCS.dll
+      #     slot = 0
+      #     "@
+      #     $pkcs11Config | Out-File -FilePath "pkcs11.cfg" -Encoding ASCII
+      #
+      #     Write-Host "PKCS#11 config created"
+      #
+      #     # Find APK files
+      #     $apkFiles = Get-ChildItem -Path "sign_binaries/android" -Recurse -Include *.apk
+      #
+      #     if ($apkFiles.Count -eq 0) {
+      #       Write-Host "No APK files found to sign"
+      #       exit 0
+      #     }
+      #
+      #     # Certum SimplySign default key alias is "1"
+      #     $keyAlias = "1"
+      #     Write-Host "Using key alias: $keyAlias"
+      #
+      #     foreach ($apk in $apkFiles) {
+      #       Write-Host "Signing APK: $($apk.FullName)"
+      #       try {
+      #         & jarsigner -verbose `
+      #           -keystore NONE `
+      #           -storetype PKCS11 `
+      #           -providerClass sun.security.pkcs11.SunPKCS11 `
+      #           -providerArg pkcs11.cfg `
+      #           -tsa http://time.certum.pl `
+      #           -signedjar "$($apk.DirectoryName)\signed-$($apk.Name)" `
+      #           "$($apk.FullName)" `
+      #           $keyAlias
+      #
+      #         # Replace original with signed version
+      #         Move-Item -Path "$($apk.DirectoryName)\signed-$($apk.Name)" -Destination $apk.FullName -Force
+      #         Write-Host "  OK: APK signed successfully"
+      #       } catch {
+      #         Write-Host "  WARNING: Failed to sign APK - $($_.Exception.Message)"
+      #       }
+      #     }
+      #
+      #     Write-Host "=== ANDROID APK SIGNING COMPLETE ==="
+      #   continue-on-error: true
+
+      - name: Verify Windows Signatures
+        shell: pwsh
+        run: |
+          $signedFiles = Get-ChildItem -Path "sign_binaries/windows" -Recurse -Include *.exe
+          foreach ($file in $signedFiles) {
+            $result = Get-AuthenticodeSignature -FilePath $file.FullName
+            $status = if ($result.Status -eq "Valid") { "VALID" } else { "INVALID/UNSIGNED ($($result.Status))" }
+            Write-Host "$($file.Name): $status"
+          }
+        continue-on-error: true
+
+      # - name: Verify APK Signature
+      #   shell: pwsh
+      #   run: |
+      #     $apkFiles = Get-ChildItem -Path "sign_binaries/android" -Recurse -Include *.apk
+      #     foreach ($apk in $apkFiles) {
+      #       Write-Host "Verifying: $($apk.Name)"
+      #       & jarsigner -verify -verbose -certs "$($apk.FullName)" 2>&1 | Select-Object -First 10
+      #     }
+      #   continue-on-error: true
+
+      - name: Upload signed Windows artifacts
+        uses: actions/upload-artifact@v4
+        with:
+          name: signed-windows-artifacts
+          path: sign_binaries/windows
+          retention-days: 7
+
+      # - name: Upload signed Android APK
+      #   uses: actions/upload-artifact@v4
+      #   with:
+      #     name: signed-android-apk
+      #     path: sign_binaries/android
+      #     retention-days: 7
+
+  release:
+    name: Create Release
+    needs: [build, sign]
+    runs-on: ubuntu-latest
+    if: always()
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v6
+
+      - name: Determine version
+        id: version
+        run: |
+          RELEASE_TYPE="${{ inputs.release_type }}"
+          if [ "$RELEASE_TYPE" = "stable" ]; then
+            VERSION="${{ inputs.version }}"
+            if [ -z "$VERSION" ]; then
+              echo "ERROR: stable release requires a version number"
+              exit 1
+            fi
+            IS_BETA="false"
+          else
+            VERSION="beta"
+            IS_BETA="true"
+          fi
+          echo "version=$VERSION" >> $GITHUB_OUTPUT
+          echo "is_beta=$IS_BETA" >> $GITHUB_OUTPUT
+          echo "Version: $VERSION, Is Beta: $IS_BETA, Release Type: $RELEASE_TYPE"
+
+      # Try to download signed Windows artifacts first, fall back to unsigned
+      - name: Download signed Windows artifacts
+        id: download-signed-win
+        uses: actions/download-artifact@v4
+        with:
+          name: signed-windows-artifacts
+          path: release_binaries
+        continue-on-error: true
+
+      - name: Download unsigned Windows artifacts (fallback)
+        if: steps.download-signed-win.outcome != 'success'
+        uses: actions/download-artifact@v4
+        with:
+          pattern: openp2p-windows-*
+          path: release_binaries
+          merge-multiple: true
+
+      # Try to download signed Android APK first, fall back to unsigned
+      # - name: Download signed Android APK
+      #   id: download-signed-android
+      #   uses: actions/download-artifact@v4
+      #   with:
+      #     name: signed-android-apk
+      #     path: release_binaries
+      #   continue-on-error: true
+      #
+      # - name: Download unsigned Android APK (fallback)
+      #   if: steps.download-signed-android.outcome != 'success'
+      #   uses: actions/download-artifact@v4
+      #   with:
+      #     name: openp2p-android-apk
+      #     path: release_binaries
+
+      - name: Download Linux artifacts
+        uses: actions/download-artifact@v4
+        with:
+          pattern: openp2p-linux-*
+          path: release_binaries
+          merge-multiple: true
+
+      - name: Download Darwin artifacts
+        uses: actions/download-artifact@v4
+        with:
+          pattern: openp2p-darwin-*
+          path: release_binaries
+          merge-multiple: true
+
+      - name: Package release assets
+        env:
+          VERSION: ${{ steps.version.outputs.version }}
+        run: |
+          mkdir -p release_assets
+          cd release_binaries
+          chmod +x * 2>/dev/null || true
+          
+          for file in *; do
+            if [ -f "$file" ]; then
+              if [[ "$file" == *.exe ]]; then
+                zip "../release_assets/${file%.exe}-${VERSION}.zip" "$file"
+              elif [[ "$file" == *.apk ]]; then
+                # APK files: rename with version, no compression needed
+                cp "$file" "../release_assets/${file%.apk}-${VERSION}.apk"
+              else
+                tar czf "../release_assets/${file}-${VERSION}.tar.gz" "$file"
+              fi
+            fi
+          done
+          
+          cd ../release_assets
+          echo "=== Release Assets ==="
+          ls -la
+
+      - name: Generate release notes
+        id: notes
+        env:
+          VERSION: ${{ steps.version.outputs.version }}
+          IS_BETA: ${{ steps.version.outputs.is_beta }}
+        run: |
+          if [ "$IS_BETA" = "true" ]; then
+            cat > release_notes.md << EOF
+          ## OpenP2P Beta Release (latest unstable)
+
+          Built from commit \`${{ github.sha }}\` on $(date -u +"%Y-%m-%d %H:%M UTC").
+
+          ### Supported Platforms
+          | OS | Architectures |
+          |---|---|
+          | Windows | amd64, arm64, i386 |
+          | Linux | amd64, arm64, i386, mips, mips64 |
+          | Darwin (macOS) | amd64, arm64 |
+          | Android | amd64, arm64 |
+
+          > **Note**: This is a pre-release build and may be unstable. This release is automatically updated on every push to master.
+          EOF
+          else
+            cat > release_notes.md << EOF
+          ## OpenP2P $VERSION
+
+          ### Supported Platforms
+          | OS | Architectures |
+          |---|---|
+          | Windows | amd64, arm64, i386 |
+          | Linux | amd64, arm64, i386, mips, mips64 |
+          | Darwin (macOS) | amd64, arm64 |
+          | Android | amd64, arm64 |
+          EOF
+          fi
+
+      - name: Delete existing Beta Release
+        if: steps.version.outputs.is_beta == 'true'
+        env:
+          GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+        run: |
+          # Delete existing beta release if it exists
+          gh release delete beta --yes --cleanup-tag 2>/dev/null || true
+
+      - name: Create Beta Release
+        if: steps.version.outputs.is_beta == 'true'
+        uses: softprops/action-gh-release@v2
+        with:
+          name: "Beta (latest unstable)"
+          tag_name: beta
+          body_path: release_notes.md
+          prerelease: true
+          files: release_assets/*
+          token: ${{ secrets.GITHUB_TOKEN }}
+
+      - name: Create Stable Release
+        if: steps.version.outputs.is_beta == 'false'
+        uses: softprops/action-gh-release@v2
+        with:
+          name: ${{ steps.version.outputs.version }}
+          tag_name: ${{ steps.version.outputs.version }}
+          body_path: release_notes.md
+          prerelease: false
+          make_latest: true
+          files: release_assets/*
+          token: ${{ secrets.GITHUB_TOKEN }}

.github/workflows/signfile.yml

@@ -0,0 +1,171 @@
+name: Sign External Binaries
+
+on:
+  workflow_dispatch:
+    inputs:
+      url_x86:
+        description: 'X86 (i386) 二进制文件下载地址'
+        type: string
+        required: false
+        default: 'https://console.openpxp.com/download/v1/latest/openp2p386-latest.exe'
+      url_x64:
+        description: 'X64 (amd64) 二进制文件下载地址'
+        type: string
+        required: false
+        default: 'https://console.openpxp.com/download/v1/latest/openp2p64-latest.exe'
+      url_arm:
+        description: 'ARM (arm64) 二进制文件下载地址'
+        type: string
+        required: false
+        default: 'https://console.openpxp.com/download/v1/latest/openp2parm64-latest.exe'
+
+permissions:
+  contents: read
+
+jobs:
+  sign:
+    name: Sign Binaries (Certum SimplySign)
+    runs-on: windows-latest
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v6
+
+      - name: Validate inputs
+        shell: bash
+        run: |
+          if [ -z "${{ inputs.url_x86 }}" ] && [ -z "${{ inputs.url_x64 }}" ] && [ -z "${{ inputs.url_arm }}" ]; then
+            echo "ERROR: 至少需要提供一个二进制文件下载地址"
+            exit 1
+          fi
+          echo "=== 输入的下载地址 ==="
+          [ -n "${{ inputs.url_x86 }}" ] && echo "X86: ${{ inputs.url_x86 }}"
+          [ -n "${{ inputs.url_x64 }}" ] && echo "X64: ${{ inputs.url_x64 }}"
+          [ -n "${{ inputs.url_arm }}" ] && echo "ARM: ${{ inputs.url_arm }}"
+
+      - name: Download binaries
+        shell: bash
+        run: |
+          mkdir -p sign_binaries
+
+          download_file() {
+            local url="$1"
+            local label="$2"
+            if [ -z "$url" ]; then
+              echo "跳过 ${label}: 未提供下载地址"
+              return
+            fi
+            echo "正在下载 ${label}: ${url}"
+            # 从 URL 中提取文件名
+            local filename=$(basename "$url" | sed 's/[?#].*//')
+            # 如果文件名为空或不合理，使用 label 作为文件名
+            if [ -z "$filename" ] || [ "$filename" = "/" ]; then
+              filename="${label}-binary.exe"
+            fi
+            curl -fSL --retry 3 --retry-delay 5 -o "sign_binaries/${filename}" "$url"
+            if [ $? -eq 0 ]; then
+              echo "下载成功: ${filename}"
+            else
+              echo "ERROR: 下载失败 ${label}: ${url}"
+              exit 1
+            fi
+          }
+
+          download_file "${{ inputs.url_x86 }}" "x86"
+          download_file "${{ inputs.url_x64 }}" "x64"
+          download_file "${{ inputs.url_arm }}" "arm"
+
+          echo ""
+          echo "=== 已下载的文件 ==="
+          ls -la sign_binaries/
+
+      - name: Setup Certum Code Signing (Windows)
+        shell: bash
+        run: |
+          echo "=== SETTING UP CERTUM CODE SIGNING FOR WINDOWS ==="
+          echo "Installing SimplySign Desktop and configuring for automatic authentication"
+
+          chmod +x ./.github/scripts/install-simplysign.sh
+          ./.github/scripts/install-simplysign.sh
+
+          echo "Configuring registry for automatic login dialog..."
+          powershell -ExecutionPolicy Bypass -File "./.github/scripts/configure-simplysign-registry.ps1"
+
+          echo "Certum signing environment ready"
+
+      - name: Authenticate Certum (Windows)
+        shell: bash
+        env:
+          CERTUM_OTP_URI: ${{ secrets.CERTUM_OTP_URI }}
+          CERTUM_USERNAME: ${{ secrets.CERTUM_USERNAME }}
+          CERTUM_CERTIFICATE_SHA1: ${{ secrets.CERTUM_CERTIFICATE_SHA1 }}
+          CERTUM_EXE_PATH: ${{ secrets.CERTUM_EXE_PATH }}
+        run: |
+          echo "=== CERTUM AUTHENTICATION ==="
+          echo "Authenticating with Certum cloud certificate using TOTP"
+
+          for attempt in 1 2 3; do
+            echo "Authentication attempt ${attempt}/3"
+            if powershell -ExecutionPolicy Bypass -File "./.github/scripts/connect-simplySign-enhanced.ps1"; then
+              echo "Authentication completed"
+              exit 0
+            fi
+
+            if [ "$attempt" -lt 3 ]; then
+              echo "Authentication attempt failed, retrying in 10 seconds..."
+              sleep 10
+            fi
+          done
+
+          echo "ERROR: Certum authentication failed after 3 attempts"
+          exit 1
+
+      - name: Sign Binaries
+        shell: bash
+        env:
+          CERTUM_CERTIFICATE_SHA1: ${{ secrets.CERTUM_CERTIFICATE_SHA1 }}
+        run: |
+          echo "=== SIGNING BINARIES ==="
+          echo "Allowing connection to stabilize..."
+          sleep 10
+
+          echo "=== PKCS#11 Library Check ==="
+          if [ -f "/c/Windows/System32/SimplySignPKCS.dll" ]; then
+            echo "PKCS#11 library present: /c/Windows/System32/SimplySignPKCS.dll"
+          else
+            echo "PKCS#11 library not found"
+          fi
+
+          echo ""
+          echo "=== SimplySign Desktop Status ==="
+          powershell -Command "
+            Write-Host 'SimplySign Desktop process status:'
+            Get-Process -Name '*SimplySign*' -ErrorAction SilentlyContinue |
+              Select-Object Name, Id, MainWindowTitle, Responding |
+              Format-Table -AutoSize
+          "
+
+          echo ""
+          echo "Proceeding to signing..."
+          echo ""
+
+          powershell -ExecutionPolicy Bypass -File "./.github/scripts/sign-windows.ps1" -TargetDirectory "sign_binaries"
+
+          echo "Binary signing completed"
+
+      - name: Verify Signatures
+        shell: pwsh
+        run: |
+          $signedFiles = Get-ChildItem -Path "sign_binaries" -Recurse -File
+          foreach ($file in $signedFiles) {
+            $result = Get-AuthenticodeSignature -FilePath $file.FullName
+            $status = if ($result.Status -eq "Valid") { "VALID" } else { "INVALID/UNSIGNED ($($result.Status))" }
+            Write-Host "$($file.Name): $status"
+          }
+        continue-on-error: true
+
+      - name: Upload signed artifacts
+        uses: actions/upload-artifact@v4
+        with:
+          name: signed-binaries
+          path: sign_binaries/
+          retention-days: 30

.gitignore

@@ -1,10 +1,8 @@
 __debug_bin
 __debug_bin.exe
 # .vscode
-test/
 openp2p.exe*
 *.log*
-go.sum
 *.tar.gz
 *.zip
 *.exe
@@ -15,4 +13,16 @@ libs/
 openp2p.app.jks
 openp2p.aar
 openp2p-sources.jar
-build.gradle
+wintun/
+wintun.dll
+.vscode/
+app/.idea/
+*_debug_bin*
+cmd/openp2p
+vendor/
+config.json
+openp2p
+lib/openp2p.dll
+cmd/config.json0
+test/docker/Dockerfile
+test/docker/get-client.sh

Changelog.md

@@ -0,0 +1,69 @@
+ChangeLog
+
+v3.25.11更新 (2026.5.15)
+Feature
+1. 
+
+Issue
+1. 修复编辑组网成员网络资源时某些情况网络不通
+1. 修复安卓版本输错token登录崩溃，下次重启自动登录仍会崩溃，不断循环
+1. 修复某些情况导致客户端异常重启
+
+v3.25.8更新 (2026.3.13)
+Feature
+1. web控制台可以修改公网监听端口
+1. 可以修改虚拟网络网段
+1. 回滚至go1.20支持老版本的macos和windows
+
+Issue
+1. 修复客户端重装后强制v6连接失效bug
+
+
+v3.25.4更新 (2026.2.9)
+Feature
+1. 优化websocket读数据卡死问题
+1. 优化睡眠唤醒客户端恢复慢问题
+
+Issue
+1. 修复获取ifconfig异常
+1. 修复数据同步异常导致设备间连接失败
+1. 修复底层连接潜在发送数据不完整问题
+
+v3.24.33更新 (2025.12.10)
+Feature
+1. 安装和升级下载文件到临时目录
+1. openwrt默认100k日志文件
+1. 使用系统dns失败时将使用223.5.5.5和8.8.8.8，安卓和部分系统有dns问题
+1. IPv6刷新时上报到服务器
+1. 设备间连接可以设置强制使用v6
+1. 编译环境使用go1.25
+
+Issue
+1. 修复加载系统证书池bug
+1. 修复初始化时崩溃
+1. 修复安卓处理多个网络资源bug
+1. 修复端口转发编辑目标设备bug
+1. 修复某些特殊情况IPv6直连失败
+
+v3.24.23更新 (2025.9.4)
+Feature
+1. 公网UDP直连
+1. upnp定期续期
+1. 支持边缘服务器
+1. 优化mp分配算法
+1. 公网IP不变不再检测nat类型
+
+Issue
+1. 修复客户端某些特殊情况卡死bug
+1. 修复wintun mtu不生效bug，增加缓冲区大小
+1. 修复广播bug
+1. 修复配置文件清空bug
+
+v3.24.13更新 (2025.6.4)
+Feature
+1. 虚拟网卡状态上报
+1. OpenWrt自动安装tun
+1. docker容器运行路径改为/usr/local/openp2p/
+
+Issue
+1. 优化客户端卡死问题

LICENSE

@@ -1,21 +1,21 @@
-MIT License
+MIT License
-
+
-Copyright (c) 2021 OpenP2P.cn
+Copyright (c) 2021 OpenP2P.cn
-
+
-Permission is hereby granted, free of charge, to any person obtaining a copy
+Permission is hereby granted, free of charge, to any person obtaining a copy
-of this software and associated documentation files (the "Software"), to deal
+of this software and associated documentation files (the "Software"), to deal
-in the Software without restriction, including without limitation the rights
+in the Software without restriction, including without limitation the rights
-to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
-copies of the Software, and to permit persons to whom the Software is
+copies of the Software, and to permit persons to whom the Software is
-furnished to do so, subject to the following conditions:
+furnished to do so, subject to the following conditions:
-
+
-The above copyright notice and this permission notice shall be included in all
+The above copyright notice and this permission notice shall be included in all
-copies or substantial portions of the Software.
+copies or substantial portions of the Software.
-
+
-THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
-IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
-FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
-AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
-LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
-OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
 SOFTWARE.

README-ZH.md

@@ -19,9 +19,9 @@
 
 [查看详细](#安全性)
 ### 4. 轻量
-文件大小2MB+，运行内存2MB+；全部在应用层实现，没有虚拟网卡，没有内核程序
+文件大小不到10MB，cpu占用极低；它可以仅跑在应用层，或者配合kmod-tun/wintun驱动使用组网功能
 ### 5. 跨平台
-因为轻量，所以很容易支持各个平台。支持主流的操作系统：Windows,Linux,MacOS；和主流的cpu架构：386、amd64、arm、arm64、mipsle、mipsle64、mips、mips64
+因为轻量，所以很容易支持各个平台。支持主流的操作系统：Windows,Linux,MacOS；和主流的cpu架构：386、amd64、arm、arm64、mipsle、mipsle64、mips、mips64、s390x、ppc64le
 ### 6. 高效
 P2P直连可以让你的设备跑满带宽。不论你的设备在任何网络环境，无论NAT1-4（Cone或Symmetric），UDP或TCP打洞,UPNP,IPv6都支持。依靠Quic协议优秀的拥塞算法，能在糟糕的网络环境获得高带宽低延时。
 
@@ -31,13 +31,13 @@ P2P直连可以让你的设备跑满带宽。不论你的设备在任何网络
 ## 快速入门
 仅需简单4步就能用起来。  
 下面是一个远程办公例子：在家里连入办公室Windows电脑。  
-（另外一个快速入门视频 https://www.bilibili.com/video/BV1Et4y1P7bF/）
+（另外一个快速入门视频 <https://www.bilibili.com/video/BV1Et4y1P7bF/>）
 ### 1.注册
-前往<https://console.openp2p.cn> 注册新用户，暂无需任何认证
+前往<https://console.openp2p.cn> 使用邮箱注册新用户，暂无需任何认证
 
    ![image](/doc/images/register.png)
 ### 2.安装
-分别在本地和远程电脑下载后双击运行，一键安装
+分别在本地和远程电脑下载后双击运行，一键安装(如果是windows用户,在浏览器下载后请勿修改文件名!!!)
 
    ![image](/doc/images/install.png)
    
@@ -46,7 +46,7 @@ Windows默认会阻止没有花钱买它家证书签名过的程序，选择“
    ![image](/doc/images/win10warn.png)
 
    ![image](/doc/images/stillrun.png)
-### 3.新建P2P应用
+### 3.新建端口转发(P2PApp)
 
 ![image](/doc/images/devices.png)
 
@@ -54,12 +54,12 @@ Windows默认会阻止没有花钱买它家证书签名过的程序，选择“
 
 ![image](/doc/images/newappedit.png)
 
-### 4.使用P2P应用
+### 4.使用端口转发(P2PApp)
-在“MyHomePC”设备上能看到刚才创建的P2P应用，连接下图显示的“本地监听端口”即可。
+在“MyHomePC2”设备上能看到刚才创建的端口转发(P2PApp)，连接下图显示的“本地监听端口”即可。
 
 ![image](/doc/images/p2pappok.png)
 
-在家里Windows电脑，按Win+R输入mstsc打开远程桌面，输入127.0.0.1:23389 /admin  
+在MyHomePC2电脑上，按Win+R输入mstsc打开远程桌面，输入127.0.0.1:23389 /admin  
 
    ![image](/doc/images/mstscconnect.png)
 
@@ -82,8 +82,8 @@ Windows默认会阻止没有花钱买它家证书签名过的程序，选择“
 ![image](/doc/images/prototype.png)
 ### 客户端架构
 ![image](/doc/images/architecture.png)
-### P2PApp
+### 端口转发(P2PApp)
-它是项目里最重要的概念，一个P2PApp就是把远程的一个服务（mstsc/ssh等）通过P2P网络映射到本地监听。二次开发或者我们提供的Restful API，主要工作就是管理P2PApp
+它是项目里最重要的概念，一个端口转发(P2PApp)就是把远程的一个服务（mstsc/ssh等）通过P2P网络映射到本地监听。二次开发或者我们提供的Restful API，主要工作就是管理端口转发(P2PApp)
 ![image](/doc/images/appdetail.png)
 ## 安全性
 加入OpenP2P共享网络的节点，只能凭授权访问。共享节点只会中转数据，别人无法访问内网任何资源。
@@ -96,29 +96,47 @@ Windows默认会阻止没有花钱买它家证书签名过的程序，选择“
 服务端有个调度模型，根据带宽、ping值、稳定性、服务时长，尽可能地使共享节点均匀地提供服务。连接共享节点使用TOTP密码，hmac-sha256算法校验，它是一次性密码，和我们平时使用的手机验证码或银行密码器一样的原理。
 
 ## 编译
-go version go1.18.1+  
+go version 1.20 only (支持win7) 
 cd到代码根目录，执行
 ```
 make
 ```
+手动编译特定系统和架构 
+All GOOS values:
+```
+"aix", "android", "darwin", "dragonfly", "freebsd", "hurd", "illumos", "ios", "js", "linux", "nacl", "netbsd", "openbsd", "plan9", "solaris", "windows", "zos"
+```
+All GOARCH values:
+```
+"386", "amd64", "amd64p32", "arm", "arm64", "arm64be", "armbe", "loong64", "mips", "mips64", "mips64le", "mips64p32", "mips64p32le", "mipsle", "ppc", "ppc64", "ppc64le", "riscv", "riscv64", "s390", "s390x", "sparc", "sparc64", "wasm"
+```
+
+比如linux+amd64
+```
+export GOPROXY=https://goproxy.io,direct
+go mod tidy
+CGO_ENABLED=0 env GOOS=linux GOARCH=amd64 go build -o openp2p --ldflags '-s -w ' -gcflags '-l' -p 8 -installsuffix cgo ./cmd
+```
 
 ## RoadMap
 近期计划：
 1. ~~支持IPv6~~(100%)
 2. ~~支持随系统自动启动，安装成系统服务~~(100%)
 3. ~~提供一些免费服务器给特别差的网络，如广电网络~~(100%)
-4. ~~建立网站，用户可以在网站管理所有P2PApp和设备。查看设备在线状态，升级，增删查改重启P2PApp等~~(100%)
+4. ~~建立网站，用户可以在网站管理所有端口转发(P2PApp)和设备。查看设备在线状态，升级，增删查改重启端口转发(P2PApp)等~~(100%)
-5. 建立公众号，用户可在微信公众号管理所有P2PApp和设备
+5. 建立公众号，用户可在微信公众号管理所有端口转发(P2PApp)和设备
 6. 客户端提供WebUI
-7. 支持自有服务器，开源服务器程序
+7. ~~支持自有服务器，开源服务器程序~~(100%)
 8. 共享节点调度模型优化，对不同的运营商优化
-9. 方便二次开发，提供API和lib
+9. ~~方便二次开发，提供API和lib~~(100%)
-10. 应用层支持UDP协议，实现很简单，但UDP应用较少暂不急(100%)
+10. ~~应用层支持UDP协议，实现很简单，但UDP应用较少暂不急~~(100%)
-11. 底层通信支持KCP协议，目前仅支持Quic；KCP专门对延时优化，被游戏加速器广泛使用，可以牺牲一定的带宽降低延时
+11. ~~底层通信支持KCP协议，目前仅支持Quic；KCP专门对延时优化，被游戏加速器广泛使用，可以牺牲一定的带宽降低延时~~(100%)
-12. 支持Android系统，让旧手机焕发青春变成移动网关
+12. ~~支持Android系统，让旧手机焕发青春变成移动网关~~(100%)
-13. 支持Windows网上邻居共享文件
+13. ~~支持Windows网上邻居共享文件~~(100%)
-14. 内网直连优化，用处不大，估计就用户测试时用到
+14. ~~内网直连优化~~(100%)
 15. ~~支持UPNP~~(100%)
+16. ~~支持Android~~(100%)
+17. 支持IOS
 
 远期计划：
 1. 利用区块链技术去中心化，让共享设备的用户有收益，从而促进更多用户共享，达到正向闭环。

README.md

@@ -19,10 +19,10 @@ The code is open source, the P2P tunnel uses TLS1.3+AES double encryption, and t
 [details](#Safety)
 
 ### 4. Lightweight
-2MB+ filesize, 2MB+ memory. It runs at appllication layer, no vitrual NIC, no kernel driver.
+10MB filesize, Extremely low CPU usage. It could only runs at application layer, or uses kmod-tun/wintun driver for SDWAN.
 
 ### 5. Cross-platform
-Benefit from lightweight, it easily supports most of major OS, like Windows, Linux, MacOS, also most of CPU architecture, like 386、amd64、arm、arm64、mipsle、mipsle64、mips、mips64.
+Benefit from lightweight, it easily supports most of major OS, like Windows, Linux, MacOS, also most of CPU architecture, like 386、amd64、arm、arm64、mipsle、mipsle64、mips、mips64、s390x、ppc64le.
 
 ### 6. Efficient
 P2P direct connection lets your devices make good use of bandwidth.  Your device can be connected in any network environments, even supports NAT1-4 (Cone or Symmetric),UDP or TCP punching,UPNP,IPv6.  Relying on the excellent congestion algorithm of the Quic protocol, high bandwidth and low latency can be obtained in a bad network environment.
@@ -35,11 +35,11 @@ Just 4 simple steps to use.
 Here's an example of remote work: connecting to an office Windows computer at home.
 (Another quick started vedio https://www.bilibili.com/video/BV1Et4y1P7bF/)
 ### 1.Register
-Go to <https://console.openp2p.cn> register a new user
+Go to <https://console.openp2p.cn> register a new user using email
 
    ![image](/doc/images/register_en.png)
 ### 2.Install
-Download on local and remote computers and double-click to run, one-click installation
+Download on local and remote computers and double-click to run, one-click installation (Windows user, please do not modify the file name after downloading in the browser!!!)
 
    ![image](/doc/images/install_en.png)
 
@@ -49,7 +49,7 @@ By default, Windows will block programs that have not been signed by the Microso
 
    ![image](/doc/images/stillrun_en.png)
    
-### 3.New P2PApp
+### 3.New Port ForWard (P2PApp)
 
 ![image](/doc/images/devices_en.png)
 
@@ -57,12 +57,12 @@ By default, Windows will block programs that have not been signed by the Microso
 
 ![image](/doc/images/newappedit_en.png)
 
-### 4.Use P2PApp
+### 4.Use Port ForWard (P2PApp)
-You can see the P2P application you just created on the "MyHomePC" device, just connect to the "local listening port" shown in the figure below.
+You can see the P2P application you just created on the "MyHomePC2" device, just connect to the "local listening port" shown in the figure below.
 
 ![image](/doc/images/p2pappok_en.png)
 
-On MyHomePC, press Win+R and enter MSTSC to open the remote desktop, input `127.0.0.1:23389 /admin`
+On MyHomePC2, press Win+R and enter MSTSC to open the remote desktop, input `127.0.0.1:23389 /admin`
 
    ![image](/doc/images/mstscconnect_en.png)
 
@@ -86,8 +86,8 @@ Especially suitable for large traffic intranet access.
 ![image](/doc/images/prototype.png)
 ### Client architecture
 ![image](/doc/images/architecture.png)
-### P2PApp
+### Port ForWard (P2PApp)
-P2PAPP is the most import concept in this project, one P2PApp is able to map the remote service(mstsc/ssh) to the local listening. The main job of re-development or restful API we provide is to manage P2PApp.
+Port ForWard (P2PApp) is the most import concept in this project, one Port ForWard (P2PApp) is able to map the remote service(mstsc/ssh) to the local listening. The main job of re-development or restful API we provide is to manage Port ForWard (P2PApp).
 
 ![image](/doc/images/appdetail.png)
 
@@ -103,30 +103,48 @@ That's right, the relay node is naturally an man-in-middle, so AES encryption is
 The server side has a scheduling model, which calculate  bandwith, ping value,stability and service duration to provide a well-proportioned service to every share node. It uses TOTP(Time-based One-time Password) with hmac-sha256 algorithem, its theory as same as the cellphone validation code or bank cipher coder.
 
 ## Build
-go version go1.18.1+  
+go version 1.20 only (support win7)
 cd root directory of the socure code and execute
 ```
 make
 ```
 
+build specified os and arch.  
+All GOOS values:
+```
+"aix", "android", "darwin", "dragonfly", "freebsd", "hurd", "illumos", "ios", "js", "linux", "nacl", "netbsd", "openbsd", "plan9", "solaris", "windows", "zos"
+```
+All GOARCH values:
+```
+"386", "amd64", "amd64p32", "arm", "arm64", "arm64be", "armbe", "loong64", "mips", "mips64", "mips64le", "mips64p32", "mips64p32le", "mipsle", "ppc", "ppc64", "ppc64le", "riscv", "riscv64", "s390", "s390x", "sparc", "sparc64", "wasm"
+```
+
+For example linux+amd64
+```
+export GOPROXY=https://goproxy.io,direct
+go mod tidy
+CGO_ENABLED=0 env GOOS=linux GOARCH=amd64 go build -o openp2p --ldflags '-s -w ' -gcflags '-l' -p 8 -installsuffix cgo ./cmd
+```
+
 ## RoadMap
 Short-Term:
 1. ~~Support IPv6.~~(100%)
 2. ~~Support auto run when system boot, setup system service.~~(100%)
 3. ~~Provide free servers to some low-performance network.~~(100%)
-4. ~~Build website, users can manage all P2PApp and devices via it. View devices' online status, upgrade, restart or CURD P2PApp .~~(100%)
+4. ~~Build website, users can manage all Port ForWard (P2PApp) and devices via it. View devices' online status, upgrade, restart or CURD Port ForWard (P2PApp) .~~(100%)
-5. Provide wechat official account, user can manage P2PApp nodes and deivce as same as website.
+5. Provide wechat official account, user can manage Port ForWard (P2PApp) nodes and deivce as same as website.
 6. Provide WebUI on client side.
-7. Support private server, open source server program.
+7. ~~Support private server, open source server program.~~(100%)
 8. Optimize our share scheduling model for different network operators.
-9. Provide REST APIs and libary for secondary development.
+9. ~~Provide REST APIs and libary for secondary development.~~(100%)
 10. ~~Support UDP at application layer, it is easy to implement but not urgent due to only a few applicaitons using UDP protocol.~~(100%)
-11. Support KCP protocol underlay, currently support Quic only. KCP focus on delay optimization,which has been widely used as game accelerator,it can sacrifice part of bandwidth to reduce timelag. 
+11. ~~Support KCP protocol underlay, currently support Quic only. KCP focus on delay optimization,which has been widely used as game accelerator,it can sacrifice part of bandwidth to reduce timelag. ~~(100%)
-12. Support Android platform, let the phones to be mobile gateway.
+12. ~~Support Android platform, let the phones to be mobile gateway.~~(100%)
-13. Support SMB Windows neighborhood.
+13. ~~Support SMB Windows neighborhood.~~(100%)
-14. Direct connection on intranet, for testing.
+14. ~~Direct connection on intranet, for testing.~~(100%)
 15. ~~Support UPNP.~~(100%)
-
+16. ~~Support Android~~(100%)
+17. Support IOS
 
 Long-Term:
 1. Use blockchain technology to decentralize, so that users who share equipment have benefits, thereby promoting more users to share, and achieving a positive closed loop.
@@ -144,3 +162,5 @@ Email: [email protected] [email protected]
 ## Disclaimer
 This project is open source for everyone to learn and use for free. It is forbidden to be used for illegal purposes. Any loss caused by improper use of this project or accident, this project and related personnel will not bear any responsibility. 
 
+## Thanks
+[![Powered by DartNode](https://dartnode.com/branding/DN-Open-Source-sm.png)](https://dartnode.com "Powered by DartNode - Free VPS for Open Source")

USAGE-ZH.md

@@ -1,99 +1,108 @@
-# 手动运行说明
+# 手动运行说明
-大部分情况通过<https://console.openp2p.cn> 操作即可。有些情况需要手动运行  
+大部分情况通过<https://console.openp2p.cn> 操作即可。有些情况需要手动运行  
-> :warning: 本文所有命令, Windows环境使用"openp2p.exe", Linux环境使用"./openp2p"
+> :warning: 本文所有命令, Windows环境使用"openp2p.exe", Linux环境使用"./openp2p"
-
+
-
+
-## 安装和监听
+## 安装和监听
-```
+```
-./openp2p install -node OFFICEPC1 -token TOKEN  
+./openp2p install -node OFFICEPC1 -token TOKEN  
-或
+或
-./openp2p -d -node OFFICEPC1 -token TOKEN  
+./openp2p -d -node OFFICEPC1 -token TOKEN  
-# 注意Windows系统把“./openp2p” 换成“openp2p.exe”
+# 注意Windows系统把“./openp2p” 换成“openp2p.exe”
-```
+```
->* install: 安装模式【推荐】，会安装成系统服务，这样它就能随系统自动启动
+>* install: 安装模式【推荐】，会安装成系统服务，这样它就能随系统自动启动
->* -d: daemon模式。发现worker进程意外退出就会自动启动新的worker进程
+>* -d: daemon模式。发现worker进程意外退出就会自动启动新的worker进程
->* -node: 独一无二的节点名字，唯一标识
+>* -node: 独一无二的节点名字，唯一标识
->* -token: 在<console.openp2p.cn>“我的”里面找到
+>* -token: 在<console.openp2p.cn>“我的”里面找到
->* -sharebandwidth: 作为共享节点时提供带宽，默认10mbps. 如果是光纤大带宽，设置越大效果越好. 0表示不共享，该节点只在私有的P2P网络使用。不加入共享的P2P网络，这样也意味着无法使用别人的共享节点
+>* -sharebandwidth: 作为共享节点时提供带宽，默认10mbps. 如果是光纤大带宽，设置越大效果越好. 0表示不共享，该节点只在私有的P2P网络使用。不加入共享的P2P网络，这样也意味着无法使用别人的共享节点
->* -loglevel: 需要查看更多调试日志，设置0；默认是1
+>* -loglevel: 需要查看更多调试日志，设置0；默认是1
-
+
-### 在docker容器里运行openp2p
+### 在docker容器里运行openp2p
-我们暂时还没提供官方docker镜像，你可以在随便一个容器里运行
+我们暂时还没提供官方docker镜像，你可以在随便一个容器里运行
-```
+```
-nohup ./openp2p -d -node OFFICEPC1 -token TOKEN  &
+nohup ./openp2p -d -node OFFICEPC1 -token TOKEN  &
-#这里由于一般的镜像都精简过，install系统服务会失败，所以使用直接daemon模式后台运行
+#这里由于一般的镜像都精简过，install系统服务会失败，所以使用直接daemon模式后台运行
-```
+```
-## 连接
+## 连接
-```
+```
-./openp2p -d -node HOMEPC123 -token TOKEN -appname OfficeWindowsRemote -peernode OFFICEPC1 -dstip 127.0.0.1 -dstport 3389 -srcport 23389
+./openp2p -d -node HOMEPC123 -token TOKEN -appname OfficeWindowsRemote -peernode OFFICEPC1 -dstip 127.0.0.1 -dstport 3389 -srcport 23389
-使用配置文件，建立多个P2PApp
+使用配置文件，建立多个P2PApp
-./openp2p -d   
+./openp2p -d   
-```
+```
->* -appname: 这个P2P应用名字
+>* -appname: 这个P2P应用名字
->* -peernode: 目标节点名字
+>* -peernode: 目标节点名字
->* -dstip: 目标服务地址，默认本机127.0.0.1
+>* -dstip: 目标服务地址，默认本机127.0.0.1
->* -dstport: 目标服务端口，常见的如windows远程桌面3389，Linux ssh 22
+>* -dstport: 目标服务端口，常见的如windows远程桌面3389，Linux ssh 22
->* -protocol: 目标服务协议 tcp、udp
+>* -protocol: 目标服务协议 tcp、udp
-
+
-## 配置文件
+## 配置文件
-一般保存在当前目录，安装模式下会保存到 `C:\Program Files\OpenP2P\config.json` 或 `/usr/local/openp2p/config.json`
+一般保存在当前目录，安装模式下会保存到 `C:\Program Files\OpenP2P\config.json` 或 `/usr/local/openp2p/config.json`
-希望修改参数，或者配置多个P2PApp可手动修改配置文件
+希望修改参数，或者配置多个P2PApp可手动修改配置文件
-
+
-配置实例
+配置实例
-```
+```
-{
+{
-  "network": {
+  "network": {
-    "Node": "hhd1207-222",
+    "Node": "YOUR-NODE-NAME",
-    "Token": "TOKEN",
+    "Token": "TOKEN",
-    "ShareBandwidth": 0,
+    "ShareBandwidth": 0,
-    "ServerHost": "api.openp2p.cn",
+    "ServerHost": "api.openp2p.cn",
-    "ServerPort": 27183,
+    "ServerPort": 27183,
-    "UDPPort1": 27182,
+    "UDPPort1": 27182,
-    "UDPPort2": 27183
+    "UDPPort2": 27183
-  },
+  },
-  "apps": [
+  "apps": [
-    {
+    {
-      "AppName": "OfficeWindowsPC",
+      "AppName": "OfficeWindowsPC",
-      "Protocol": "tcp",
+      "Protocol": "tcp",
-      "SrcPort": 23389,
+      "SrcPort": 23389,
-      "PeerNode": "OFFICEPC1",
+      "PeerNode": "OFFICEPC1",
-      "DstPort": 3389,
+      "DstPort": 3389,
-      "DstHost": "localhost",
+      "DstHost": "localhost",
-    },
+    },
-    {
+    {
-      "AppName": "OfficeServerSSH",
+      "AppName": "OfficeServerSSH",
-      "Protocol": "tcp",
+      "Protocol": "tcp",
-      "SrcPort": 22,
+      "SrcPort": 22,
-      "PeerNode": "OFFICEPC1",
+      "PeerNode": "OFFICEPC1",
-      "DstPort": 22,
+      "DstPort": 22,
-      "DstHost": "192.168.1.5",
+      "DstHost": "192.168.1.5",
-    }
+    }
-  ]
+  ]
-}
+}
-```
+```
-
+
-## 升级客户端
+## 升级客户端
-```
+```
-# update local client
+# update local client
-./openp2p update  
+./openp2p update  
-# update remote client
+# update remote client
-curl --insecure 'https://api.openp2p.cn:27183/api/v1/device/YOUR-NODE-NAME/update?user=&password='
+curl --insecure 'https://api.openp2p.cn:27183/api/v1/device/YOUR-NODE-NAME/update?user=&password='
-```
+```
-
+
-Windows系统需要设置防火墙放行本程序，程序会自动设置，如果设置失败会影响连接功能。
+Windows系统需要设置防火墙放行本程序，程序会自动设置，如果设置失败会影响连接功能。
-Linux系统（Ubuntu和CentOS7）的防火墙默认配置均不会有影响，如果不行可尝试关闭防火墙
+Linux系统（Ubuntu和CentOS7）的防火墙默认配置均不会有影响，如果不行可尝试关闭防火墙
-```
+```
-systemctl stop firewalld.service
+systemctl stop firewalld.service
-systemctl start firewalld.service
+systemctl start firewalld.service
-firewall-cmd --state
+firewall-cmd --state
-```
+```
-
+## 停止
-## 卸载
+TODO: windows linux macos
-```
+## 卸载
-./openp2p uninstall
+```
-# 已安装时
+./openp2p uninstall
-# windows
+# 已安装时
-C:\Program Files\OpenP2P\openp2p.exe uninstall
+# windows
-# linux,macos
+C:\Program Files\OpenP2P\openp2p.exe uninstall
-sudo /usr/local/openp2p/openp2p uninstall
+# linux,macos
-```
+sudo /usr/local/openp2p/openp2p uninstall
+```
+
+## Docker运行
+```
+# 把YOUR-TOKEN和YOUR-NODE-NAME替换成自己的
+docker run -d --restart=always --net host --name openp2p-client -e OPENP2P_TOKEN=YOUR-TOKEN -e OPENP2P_NODE=YOUR-NODE-NAME  openp2pcn/openp2p-client:latest 
+OR
+docker run -d --restart=always --net host --name openp2p-client  openp2pcn/openp2p-client:latest -token YOUR-TOKEN -node YOUR-NODE-NAME
+```

USAGE.md

@@ -1,101 +1,109 @@
-
+
-
+
-# Parameters details
+# Parameters details
-In most cases, you can operate it through <https://console.openp2p.cn>. In some cases it is necessary to run manually
+In most cases, you can operate it through <https://console.openp2p.cn>. In some cases it is necessary to run manually
-> :warning: all commands in this doc, Windows env uses "openp2p.exe", Linux env uses "./openp2p" 
+> :warning: all commands in this doc, Windows env uses "openp2p.exe", Linux env uses "./openp2p" 
-
+
-
+
-## Install and Listen
+## Install and Listen
-```
+```
-./openp2p install -node OFFICEPC1 -token TOKEN  
+./openp2p install -node OFFICEPC1 -token TOKEN  
-Or
+Or
-./openp2p -d -node OFFICEPC1 -token TOKEN  
+./openp2p -d -node OFFICEPC1 -token TOKEN  
-
+
-```
+```
->* install: [recommand] will install as system service. So it will autorun when system booting.
+>* install: [recommand] will install as system service. So it will autorun when system booting.
->* -d: daemon mode run once. When the worker process is found to exit unexpectedly, a new worker process will be automatically started
+>* -d: daemon mode run once. When the worker process is found to exit unexpectedly, a new worker process will be automatically started
->* -node: Unique node name, unique identification
+>* -node: Unique node name, unique identification
->* -token: See <console.openp2p.cn> "Profile"
+>* -token: See <console.openp2p.cn> "Profile"
->* -sharebandwidth: Provides bandwidth when used as a shared node, the default is 10mbps. If it is a large bandwidth of optical fiber, the larger the setting, the better the effect. 0 means not shared, the node is only used in a private P2P network. Do not join the shared P2P network, which also means that you CAN NOT use other people’s shared nodes
+>* -sharebandwidth: Provides bandwidth when used as a shared node, the default is 10mbps. If it is a large bandwidth of optical fiber, the larger the setting, the better the effect. 0 means not shared, the node is only used in a private P2P network. Do not join the shared P2P network, which also means that you CAN NOT use other people’s shared nodes
->* -loglevel: Need to view more debug logs, set 0; the default is 1
+>* -loglevel: Need to view more debug logs, set 0; the default is 1
-
+
-### Run in Docker container
+### Run in Docker container
-We don't provide official docker image yet, you can run it in any container
+We don't provide official docker image yet, you can run it in any container
-```
+```
-nohup ./openp2p -d -node OFFICEPC1 -token TOKEN  &
+nohup ./openp2p -d -node OFFICEPC1 -token TOKEN  &
-# Since many docker images have been simplified, the install system service will fail, so the daemon mode is used to run in the background
+# Since many docker images have been simplified, the install system service will fail, so the daemon mode is used to run in the background
-```
+```
-
+
-## Connect
+## Connect
-```
+```
-./openp2p -d -node HOMEPC123 -token TOKEN -appname OfficeWindowsRemote -peernode OFFICEPC1 -dstip 127.0.0.1 -dstport 3389 -srcport 23389
+./openp2p -d -node HOMEPC123 -token TOKEN -appname OfficeWindowsRemote -peernode OFFICEPC1 -dstip 127.0.0.1 -dstport 3389 -srcport 23389
-Create multiple P2PApp by config file
+Create multiple P2PApp by config file
-./openp2p -d    
+./openp2p -d    
-```
+```
->* -appname: This P2PApp name
+>* -appname: This P2PApp name
->* -peernode: Target node name
+>* -peernode: Target node name
->* -dstip: Target service address, default local 127.0.0.1
+>* -dstip: Target service address, default local 127.0.0.1
->* -dstport: Target service port, such as windows remote desktop 3389, Linux ssh 22
+>* -dstport: Target service port, such as windows remote desktop 3389, Linux ssh 22
->* -protocol: Target service protocol tcp, udp
+>* -protocol: Target service protocol tcp, udp
-
+
-## Config file
+## Config file
-Generally saved in the current directory, in installation mode it will be saved to `C:\Program Files\OpenP2P\config.json` or `/usr/local/openp2p/config.json`
+Generally saved in the current directory, in installation mode it will be saved to `C:\Program Files\OpenP2P\config.json` or `/usr/local/openp2p/config.json`
-If you want to modify the parameters, or configure multiple P2PApps, you can manually modify the configuration file
+If you want to modify the parameters, or configure multiple P2PApps, you can manually modify the configuration file
-
+
-Configuration example
+Configuration example
-```
+```
-{
+{
-  "network": {
+  "network": {
-    "Node": "hhd1207-222",
+    "Node": "YOUR-NODE-NAME",
-    "Token": "TOKEN",
+    "Token": "TOKEN",
-    "ShareBandwidth": 0,
+    "ShareBandwidth": 0,
-    "ServerHost": "api.openp2p.cn",
+    "ServerHost": "api.openp2p.cn",
-    "ServerPort": 27183,
+    "ServerPort": 27183,
-    "UDPPort1": 27182,
+    "UDPPort1": 27182,
-    "UDPPort2": 27183
+    "UDPPort2": 27183
-  },
+  },
-  "apps": [
+  "apps": [
-    {
+    {
-      "AppName": "OfficeWindowsPC",
+      "AppName": "OfficeWindowsPC",
-      "Protocol": "tcp",
+      "Protocol": "tcp",
-      "SrcPort": 23389,
+      "SrcPort": 23389,
-      "PeerNode": "OFFICEPC1",
+      "PeerNode": "OFFICEPC1",
-      "DstPort": 3389,
+      "DstPort": 3389,
-      "DstHost": "localhost",
+      "DstHost": "localhost",
-    },
+    },
-    {
+    {
-      "AppName": "OfficeServerSSH",
+      "AppName": "OfficeServerSSH",
-      "Protocol": "tcp",
+      "Protocol": "tcp",
-      "SrcPort": 22,
+      "SrcPort": 22,
-      "PeerNode": "OFFICEPC1",
+      "PeerNode": "OFFICEPC1",
-      "DstPort": 22,
+      "DstPort": 22,
-      "DstHost": "192.168.1.5",
+      "DstHost": "192.168.1.5",
-    }
+    }
-  ]
+  ]
-}
+}
-```
+```
-## Client update
+## Client update
-```
+```
-# update local client
+# update local client
-./openp2p update  
+./openp2p update  
-# update remote client
+# update remote client
-curl --insecure 'https://api.openp2p.cn:27183/api/v1/device/YOUR-NODE-NAME/update?user=&password='
+curl --insecure 'https://api.openp2p.cn:27183/api/v1/device/YOUR-NODE-NAME/update?user=&password='
-```
+```
-
+
-Windows system needs to set up firewall for this program, the program will automatically set the firewall, if the setting fails, the UDP punching will be affected.  
+Windows system needs to set up firewall for this program, the program will automatically set the firewall, if the setting fails, the UDP punching will be affected.  
-The default firewall configuration of Linux system (Ubuntu and CentOS7) will not have any effect, if not, you can try to turn off the firewall
+The default firewall configuration of Linux system (Ubuntu and CentOS7) will not have any effect, if not, you can try to turn off the firewall
-```
+```
-systemctl stop firewalld.service
+systemctl stop firewalld.service
-systemctl start firewalld.service
+systemctl start firewalld.service
-firewall-cmd --state
+firewall-cmd --state
-```
+```
-
+
-## Uninstall
+## Uninstall
-```
+```
-./openp2p uninstall
+./openp2p uninstall
-# when already installed
+# when already installed
-# windows
+# windows
-C:\Program Files\OpenP2P\openp2p.exe uninstall
+C:\Program Files\OpenP2P\openp2p.exe uninstall
-# linux,macos
+# linux,macos
-sudo /usr/local/openp2p/openp2p uninstall
+sudo /usr/local/openp2p/openp2p uninstall
+```
+
+## Run with Docker
+```
+# Replace YOUR-TOKEN and YOUR-NODE-NAME with yours
+docker run -d --net host --name openp2p-client -e OPENP2P_TOKEN=YOUR-TOKEN -e OPENP2P_NODE=YOUR-NODE-NAME  openp2pcn/openp2p-client:latest 
+OR
+docker run -d --net host --name openp2p-client  openp2pcn/openp2p-client:latest -token YOUR-TOKEN -node YOUR-NODE-NAME
 ```

app/.idea/runConfigurations.xml

@@ -1,10 +0,0 @@
-<?xml version="1.0" encoding="UTF-8"?>
-<project version="4">
-  <component name="RunConfigurationProducerService">
-    <option name="ignoredProducers">
-      <set>
-        <option value="com.android.tools.idea.compose.preview.runconfiguration.ComposePreviewRunConfigurationProducer" />
-      </set>
-    </option>
-  </component>
-</project>

app/README.md

@@ -0,0 +1,33 @@
+## Build
+depends on openjdk 11, gradle 8.1.3, ndk 21
+```
+
+# latest version not support go1.20
+go install golang.org/x/mobile/cmd/gomobile@7c4916698cc93475ebfea76748ee0faba2deb2a5
+gomobile init
+go get -v golang.org/x/mobile/bind@7c4916698cc93475ebfea76748ee0faba2deb2a5
+cd core
+gomobile bind -target android -v
+if [[ $? -ne 0 ]]; then
+    echo "build error"
+    exit 9
+fi
+echo "build ok"
+cp openp2p.aar openp2p-sources.jar ../app/app/libs
+echo "copy to APP libs"
+
+edit app/app/build.gradle 
+```
+signingConfigs {
+        release {
+            storeFile file('YOUR-JKS-PATH')
+            storePassword 'YOUR-PASSWORD'
+            keyAlias 'openp2p.keys'
+            keyPassword 'YOUR-PASSWORD'
+        }
+    }
+```
+cd ../app
+./gradlew build
+
+```

app/app/build.gradle

@@ -0,0 +1,63 @@
+plugins {
+    id 'com.android.application'
+    id 'kotlin-android'
+}
+
+android {
+    signingConfigs {
+        release {
+            storeFile file('C:\\work\\src\\openp2p-client\\app\\openp2p.jks')
+            storePassword 'YOUR-PASSWORD'
+            keyAlias 'openp2p.keys'
+            keyPassword 'YOUR-PASSWORD'
+        }
+    }
+    compileSdkVersion 31
+    buildToolsVersion "30.0.3"
+
+    defaultConfig {
+        applicationId "cn.openp2p"
+        minSdkVersion 16
+        targetSdkVersion 31
+        versionCode 1
+        versionName "2718281828"
+
+        testInstrumentationRunner "androidx.test.runner.AndroidJUnitRunner"
+        namespace "cn.openp2p"
+    }
+
+    buildTypes {
+        release {
+            minifyEnabled true
+            shrinkResources true
+            proguardFiles getDefaultProguardFile('proguard-android-optimize.txt'), 'proguard-rules.pro'
+            signingConfig signingConfigs.release
+        }
+    }
+    compileOptions {
+        sourceCompatibility JavaVersion.VERSION_1_8
+        targetCompatibility JavaVersion.VERSION_1_8
+    }
+    kotlinOptions {
+        jvmTarget = '1.8'
+    }
+    buildFeatures {
+        viewBinding true
+    }
+}
+
+dependencies {
+    implementation fileTree(dir: "libs", include: ["*.jar", "*.aar"])
+    implementation "org.jetbrains.kotlin:kotlin-stdlib:$kotlin_version"
+    implementation 'androidx.core:core-ktx:1.3.1'
+    implementation 'androidx.appcompat:appcompat:1.2.0'
+    implementation 'com.google.android.material:material:1.2.1'
+    implementation 'androidx.annotation:annotation:1.1.0'
+    implementation 'androidx.constraintlayout:constraintlayout:2.0.1'
+    implementation 'androidx.lifecycle:lifecycle-livedata-ktx:2.2.0'
+    implementation 'androidx.lifecycle:lifecycle-viewmodel-ktx:2.2.0'
+    testImplementation 'junit:junit:4.+'
+    androidTestImplementation 'androidx.test.ext:junit:1.1.2'
+    androidTestImplementation 'androidx.test.espresso:espresso-core:3.3.0'
+    implementation files('libs\\openp2p-sources.jar')
+}

app/app/src/main/AndroidManifest.xml

@@ -1,12 +1,11 @@
 <?xml version="1.0" encoding="utf-8"?>
-<manifest xmlns:android="http://schemas.android.com/apk/res/android"
+<manifest xmlns:android="http://schemas.android.com/apk/res/android">
-    package="cn.openp2p">
 
     <uses-permission android:name="android.permission.FOREGROUND_SERVICE" />
     <uses-permission android:name="android.permission.INTERNET" />
     <uses-permission android:name="android.permission.WRITE_EXTERNAL_STORAGE" />
     <uses-permission android:name="android.permission.READ_EXTERNAL_STORAGE" />
-
+    <uses-permission android:name="android.permission.RECEIVE_BOOT_COMPLETED" />
 
     <application
         android:allowBackup="true"
@@ -27,13 +26,21 @@
 
         <activity
             android:name=".ui.login.LoginActivity"
-            android:label="@string/app_name">
+            android:label="@string/app_name"
+            android:exported="true">
             <intent-filter>
                 <action android:name="android.intent.action.MAIN" />
 
                 <category android:name="android.intent.category.LAUNCHER" />
             </intent-filter>
         </activity>
+        <receiver android:name="BootReceiver"
+            android:exported="true"
+            android:enabled="true">
+            <intent-filter>
+                <action android:name="android.intent.action.BOOT_COMPLETED"/>
+            </intent-filter>
+        </receiver>
     </application>
 
 </manifest>

app/app/src/main/java/cn/openp2p/BootReceiver.kt

@@ -0,0 +1,27 @@
+package cn.openp2p
+
+import android.content.BroadcastReceiver
+import android.content.Context
+import android.content.Intent
+import android.net.VpnService
+import android.os.Build
+import android.os.Bundle
+import android.util.Log
+import cn.openp2p.ui.login.LoginActivity
+class BootReceiver : BroadcastReceiver() {
+    override fun onReceive(context: Context, intent: Intent) {
+//        Logger.log("pp onReceive "+intent.action.toString())
+        Log.i("onReceive","start "+intent.action.toString())
+        // if (Intent.ACTION_BOOT_COMPLETED == intent.action) {
+        //     Log.i("onReceive","match "+intent.action.toString())
+        //     VpnService.prepare(context)
+        //     val intent = Intent(context, OpenP2PService::class.java)
+        //     if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.O) {
+        //         context.startForegroundService(intent)
+        //     } else {
+        //         context.startService(intent)
+        //     }
+        // }
+        Log.i("onReceive","end "+intent.action.toString())
+    }
+}

app/app/src/main/java/cn/openp2p/OpenP2PService.kt

@@ -4,9 +4,11 @@ import android.app.*
 import android.content.Context
 import android.content.Intent
 import android.graphics.Color
+import android.net.VpnService
 import android.os.Binder
 import android.os.Build
 import android.os.IBinder
+import android.os.ParcelFileDescriptor
 import android.util.Log
 import androidx.annotation.RequiresApi
 import androidx.core.app.NotificationCompat
@@ -14,11 +16,32 @@ import cn.openp2p.ui.login.LoginActivity
 import kotlinx.coroutines.GlobalScope
 import kotlinx.coroutines.launch
 import openp2p.Openp2p
+import java.io.FileInputStream
+import java.io.FileOutputStream
+import java.nio.ByteBuffer
+import kotlinx.coroutines.*
+import org.json.JSONObject
+import java.io.File
+import java.net.InetAddress
+import java.net.NetworkInterface
+import kotlinx.coroutines.channels.Channel
+import java.nio.channels.FileChannel
+import kotlinx.coroutines.Dispatchers
+import kotlinx.coroutines.withContext
+
+data class Node(val name: String, val ip: String, val resource: String? = null)
 
 
-class OpenP2PService : Service() {
+data class Network(
+    val id: Long,
+    val name: String,
+    val gateway: String,
+    val Nodes: List<Node>
+)
+
+class OpenP2PService : VpnService() {
     companion object {
-        private val LOG_TAG = OpenP2PService::class.simpleName
+        private val LOG_TAG = "OpenP2PService"
     }
 
     inner class LocalBinder : Binder() {
@@ -28,11 +51,18 @@ class OpenP2PService : Service() {
     private val binder = LocalBinder()
     private lateinit var network: openp2p.P2PNetwork
     private lateinit var mToken: String
-    private var running:Boolean =true
+    private var running: Boolean = true
+    private var sdwanRunning: Boolean = false
+    private var vpnInterface: ParcelFileDescriptor? = null
+    private var sdwanJob: Job? = null
+    private val packetQueue = Channel<ByteBuffer>(capacity = 1024)
+    private val serviceScope = CoroutineScope(SupervisorJob() + Dispatchers.IO)
+
     override fun onCreate() {
-        Log.i(LOG_TAG, "onCreate - Thread ID = " + Thread.currentThread().id)
+        val logDir = File(getExternalFilesDir(null), "log")
-        var channelId: String? = null
+        Logger.init(logDir)
-        channelId = if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.O) {
+        Logger.i(LOG_TAG, "onCreate - Thread ID = " + Thread.currentThread().id)
+        var channelId = if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.O) {
             createNotificationChannel("kim.hsl", "ForegroundService")
         } else {
             ""
@@ -41,12 +71,12 @@ class OpenP2PService : Service() {
 
         val pendingIntent = PendingIntent.getActivity(
             this, 0,
-            notificationIntent, 0
+            notificationIntent, PendingIntent.FLAG_IMMUTABLE
         )
 
         val notification = channelId?.let {
             NotificationCompat.Builder(this, it)
-    //            .setSmallIcon(R.mipmap.app_icon)
+                //            .setSmallIcon(R.mipmap.app_icon)
                 .setContentTitle("My Awesome App")
                 .setContentText("Doing some work...")
                 .setContentIntent(pendingIntent).build()
@@ -54,52 +84,308 @@ class OpenP2PService : Service() {
 
         startForeground(1337, notification)
         super.onCreate()
-
+        refreshSDWAN()
     }
 
     override fun onStartCommand(intent: Intent?, flags: Int, startId: Int): Int {
-        Log.i(
+        Logger.i(
             LOG_TAG,
             "onStartCommand - startId = " + startId + ", Thread ID = " + Thread.currentThread().id
         )
+        startOpenP2P(null)
         return super.onStartCommand(intent, flags, startId)
     }
 
     override fun onBind(p0: Intent?): IBinder? {
-
         val token = p0?.getStringExtra("token")
-        Log.i(LOG_TAG, "onBind - Thread ID = " + Thread.currentThread().id + token)
+        Logger.i(LOG_TAG, "onBind token=$token")
-        GlobalScope.launch {
+        startOpenP2P(token)
-            network = Openp2p.runAsModule(getExternalFilesDir(null).toString(), token, 0, 1)
-            val isConnect = network.connect(30000)  // ms
-            Log.i(OpenP2PService.LOG_TAG, "login result: " + isConnect.toString());
-            do {
-                Thread.sleep(1000)
-            }while(network.connect(30000)&&running)
-            stopSelf()
-        }
         return binder
     }
 
+    private fun startOpenP2P(token: String?): Boolean {
+        if (sdwanRunning) {
+            return true
+        }
+
+        Logger.i(LOG_TAG, "startOpenP2P - Thread ID = " + Thread.currentThread().id + token)
+        val oldToken = Openp2p.getToken(getExternalFilesDir(null).toString())
+        Logger.i(LOG_TAG, "startOpenP2P oldtoken=$oldToken newtoken=$token")
+        if (oldToken == "0" && token == null) {
+            return false
+        }
+        sdwanRunning = true
+        //        runSDWAN()
+        GlobalScope.launch {
+            network = Openp2p.runAsModule(
+                getExternalFilesDir(null).toString(),
+                token,
+                0,
+                1
+            ) // /storage/emulated/0/Android/data/cn.openp2p/files/
+            val isConnect = network.connect(30000)  // ms
+            Logger.i(LOG_TAG, "login result: " + isConnect.toString());
+            do {
+                Thread.sleep(1000)
+            } while (network.connect(30000) && running)
+            stopSelf()
+        }
+        return false
+    }
+
+    private fun refreshSDWAN() {
+        GlobalScope.launch {
+            Logger.i(OpenP2PService.LOG_TAG, "refreshSDWAN start");
+            while (true) {
+                Logger.i(OpenP2PService.LOG_TAG, "waiting new sdwan config");
+                val buf = ByteArray(32 * 1024)
+                val buffLen = Openp2p.getAndroidSDWANConfig(buf)
+                Logger.i(OpenP2PService.LOG_TAG, "closing running sdwan instance");
+                sdwanRunning = false
+                vpnInterface?.close()
+                vpnInterface = null
+                sdwanJob?.join()
+                sdwanJob = serviceScope.launch(context = Dispatchers.IO) {
+                    runSDWAN(buf.copyOfRange(0, buffLen.toInt()))
+                }
+            }
+            Logger.i(OpenP2PService.LOG_TAG, "refreshSDWAN end");
+        }
+    }
+
+    private suspend fun readTunLoop() {
+        val inputStream = FileInputStream(vpnInterface?.fileDescriptor).channel
+        if (inputStream == null) {
+            Logger.i(OpenP2PService.LOG_TAG, "open FileInputStream error: ");
+            return
+        }
+        Logger.i(LOG_TAG, "read tun loop start")
+        val buffer = ByteBuffer.allocate(4096)
+        val byteArrayRead = ByteArray(4096)
+        while (sdwanRunning) {
+            buffer.clear()
+            withContext(Dispatchers.IO) {
+                val readBytes = inputStream.read(buffer)
+                if (readBytes > 0) {
+                    buffer.flip()
+                    buffer.get(byteArrayRead, 0, readBytes)
+                    // Logger.i(OpenP2PService.LOG_TAG, String.format("Openp2p.androidRead: %d", readBytes))
+                    Openp2p.androidRead(byteArrayRead, readBytes.toLong())
+                    // Logger.i(OpenP2PService.LOG_TAG, "inputStream.read error: ")
+                } else {
+                    delay(50)
+                }
+            }
+        }
+        Logger.i(LOG_TAG, "read tun loop end")
+    }
+
+
+    private suspend fun runSDWAN(buf: ByteArray) {
+//        val localIps = listOf(
+//            "fe80::14b6:a0ff:fe3e:64de" to 64,
+//            "192.168.100.184" to 24,
+//            "10.93.158.91" to 32,
+//            "192.168.3.66" to 24
+//        )
+//
+//        // 测试用例
+//        val testCases = listOf(
+//            "192.168.3.11" to true,
+//            "192.168.100.1" to true,
+//            "192.168.101.1" to false,
+//            "10.93.158.91" to true,
+//            "10.93.158.90" to false,
+//            "fe80::14b6:a0ff:fe3e:64de" to true,
+//            "fe80::14b6:a0ff:fe3e:64dd" to true // 在同一子网
+//        )
+//
+//        for ((ip, expected) in testCases) {
+//            val result = isSameSubnet(ip, localIps)
+//            println("Testing IP: $ip, Expected: $expected, Result: $result")
+//        }
+        sdwanRunning = true
+
+        Logger.i(OpenP2PService.LOG_TAG, "runSDWAN start:${buf.decodeToString()}");
+        try {
+            var builder = Builder()
+           val jsonObject = JSONObject(buf.decodeToString())
+            // debug sdwan info
+            // val jsonObject = JSONObject("""{"id":2817104318517097000,"name":"network1","gateway":"10.2.3.254/24","mode":"central","centralNode":"nanjin-192-168-0-82","enable":1,"tunnelNum":3,"mtu":1420,"Nodes":[{"name":"192-168-24-15","ip":"10.2.3.5"},{"name":"Alpine Linux-172.16","ip":"10.2.3.14","resource":"172.16.0.0/24"},{"name":"ctdeMacBook-Pro.local","ip":"10.2.3.22"},{"name":"dengjiandeMBP.sh.chaitin.net","ip":"10.2.3.32"},{"name":"DESKTOP-WIN11-ARM-self","ip":"10.2.3.19"},{"name":"eastdeMBP.sh.chaitin.net","ip":"10.2.3.3"},{"name":"FN-NAS-HP","ip":"10.2.3.1","resource":"192.168.100.0/24"},{"name":"huangruideMBP.sh.chaitin.net","ip":"10.2.3.30"},{"name":"iStoreOS-virtual-machine","ip":"10.2.3.12"},{"name":"k30s-redmi-10.2.33","ip":"10.2.3.27"},{"name":"lincheng-MacBook-Pro-3.sh.chaitin.net","ip":"10.2.3.15"},{"name":"localhost-mi-13","ip":"10.2.3.8"},{"name":"localhost-华为matepad11","ip":"10.2.3.13"},{"name":"luzhanwendeMacBook-Pro.local","ip":"10.2.3.17"},{"name":"Mi-pad2-local","ip":"10.2.3.9"},{"name":"nanjin-192-168-0-82","ip":"10.2.3.34"},{"name":"R7000P-2021","ip":"10.2.3.7"},{"name":"tanxiaolongsMBP.sh.chaitin.net","ip":"10.2.3.20"},{"name":"TUF-AX3000_V2-3804","ip":"10.2.3.25"},{"name":"WIN-CYZ-10.2.3.16","ip":"10.2.3.16"},{"name":"WODOUYAO","ip":"10.2.3.4"},{"name":"Zstrack01","ip":"10.2.3.51","resource":"192.168.24.0/22,192.168.20.0/24"},{"name":"小米14-localhost","ip":"10.2.3.23"}]}""")
+            val id = jsonObject.getLong("id")
+            val mtu = jsonObject.getInt("mtu")
+            val name = jsonObject.getString("name")
+            val gateway = jsonObject.getString("gateway")
+            val nodesArray = jsonObject.getJSONArray("Nodes")
+
+            val nodesList = mutableListOf<JSONObject>()
+            for (i in 0 until nodesArray.length()) {
+                nodesList.add(nodesArray.getJSONObject(i))
+            }
+
+            val myNodeName = Openp2p.getAndroidNodeName()
+            // 使用本地 IP 和子网判断是否需要添加路由
+            val localIps = getLocalIpAndSubnet()
+            Logger.i(OpenP2PService.LOG_TAG, "getAndroidNodeName:${myNodeName}");
+            val nodeList = nodesList.map {
+                val nodeName = it.getString("name")
+                val nodeIp = it.getString("ip")
+                if (nodeName == myNodeName) {
+                    try {
+                        Logger.i(LOG_TAG, "Attempting to add address: $nodeIp/24")
+                        builder.addAddress(nodeIp, 24)
+                        Logger.i(LOG_TAG, "Successfully added address")
+                    } catch (e: Exception) {
+                        Logger.e(LOG_TAG, "Failed to add address $nodeIp: ${e.message}")
+                        throw e // or handle gracefully
+                    }
+                }
+                val nodeResource = it.optString("resource", null)
+                if (!nodeResource.isNullOrEmpty()) {
+                    // 可能是多个网段，用逗号分隔
+                    val resourceList = nodeResource.split(",")
+                    for (resource in resourceList) {
+                        val parts = resource.split("/")
+                        if (parts.size == 2) {
+                            val ipAddress = parts[0].trim()
+                            val subnetMask = parts[1].trim()
+                            // 判断是否属于本机网段
+                            if (!isSameSubnet(ipAddress, localIps)) {
+                                builder.addRoute(ipAddress, subnetMask.toInt())
+                                Logger.i(
+                                    OpenP2PService.LOG_TAG,
+                                    "sdwan addRoute:${ipAddress},${subnetMask}"
+                                )
+                            } else {
+                                Logger.i(
+                                    OpenP2PService.LOG_TAG,
+                                    "Skipped adding route for ${ipAddress}, already in local subnet"
+                                )
+                            }
+                        } else {
+                            Logger.w(OpenP2PService.LOG_TAG, "Invalid resource format: $resource")
+                        }
+                    }
+                }
+
+                Node(nodeName, nodeIp, nodeResource)
+            }
+
+            val network = Network(id, name, gateway, nodeList)
+            println(network)
+            Logger.i(OpenP2PService.LOG_TAG, "onBind");
+            builder.addDnsServer("119.29.29.29")
+            builder.addDnsServer("2400:3200::1") // alicloud dns v6 & v4
+            // builder.addRoute("10.2.3.0", 24)
+//            builder.addRoute("0.0.0.0", 0);
+            val gatewayStr = jsonObject.optString("gateway", "")
+            val subNet = getNetworkAddress(gatewayStr)
+            if (subNet != null) {
+                val (netIp, prefix) = subNet
+                builder.addRoute(netIp, prefix)
+                Logger.i(OpenP2PService.LOG_TAG, "Added route from gateway: $netIp/$prefix")
+            } else {
+                Logger.w(OpenP2PService.LOG_TAG, "Invalid gateway format: $gatewayStr")
+            }
+
+            builder.setSession(LOG_TAG!!)
+            builder.setMtu(mtu)
+            vpnInterface = builder.establish()
+            if (vpnInterface == null) {
+                Log.e(OpenP2PService.LOG_TAG, "start vpnservice error: ");
+            }
+
+            val byteArrayWrite = ByteArray(4096)
+            serviceScope.launch(Dispatchers.IO) {
+                readTunLoop() // 文件读操作，适合 Dispatchers.IO
+            }
+
+            val outputStream = FileOutputStream(vpnInterface?.fileDescriptor).channel
+            if (outputStream == null) {
+                Log.e(OpenP2PService.LOG_TAG, "open FileOutputStream error: ");
+                return
+            }
+            Logger.i(LOG_TAG, "write tun loop start")
+            while (sdwanRunning) {
+                val len = Openp2p.androidWrite(byteArrayWrite, 3000)
+                if (len > mtu || len.toInt() == 0) {
+                    continue
+                }
+                try {
+                    val writeBytes =
+                        outputStream?.write(ByteBuffer.wrap(byteArrayWrite, 0, len.toInt()))
+                    if (writeBytes != null && writeBytes <= 0) {
+                        Logger.e(LOG_TAG, "outputStream.write failed: $writeBytes")
+                    }
+                } catch (e: Exception) {
+                    Logger.e(LOG_TAG, "outputStream.write exception: ${e.message}")
+                    e.printStackTrace()
+                    continue
+                }
+            }
+            outputStream.close()
+            vpnInterface?.close()
+            vpnInterface = null
+            Logger.i(LOG_TAG, "write tun loop end")
+        } catch (e: Exception) {
+            Logger.i("VPN Connection", "发生异常: ${e.message}")
+        }
+        Logger.i(OpenP2PService.LOG_TAG, "runSDWAN end");
+    }
+    /**
+     * 将 "10.2.3.254/16" 这样的 CIDR 转成正确对齐的网络地址，如 "10.2.0.0/16"
+     */
+    fun getNetworkAddress(cidr: String): Pair<String, Int>? {
+        val parts = cidr.trim().split("/")
+        if (parts.size != 2) return null
+
+        val ip = parts[0]
+        val prefix = parts[1].toIntOrNull() ?: return null
+        if (prefix !in 0..32) return null
+
+        val octets = ip.split(".").map { it.toInt() }
+        if (octets.size != 4) return null
+
+        // 转成整数
+        val ipInt = (octets[0] shl 24) or (octets[1] shl 16) or (octets[2] shl 8) or octets[3]
+
+        // 生成掩码并计算网络地址
+        val mask = if (prefix == 0) 0 else (-1 shl (32 - prefix))
+        val networkInt = ipInt and mask
+
+        // 转回点分十进制
+        val networkIp = listOf(
+            (networkInt shr 24) and 0xFF,
+            (networkInt shr 16) and 0xFF,
+            (networkInt shr 8) and 0xFF,
+            networkInt and 0xFF
+        ).joinToString(".")
+
+        return networkIp to prefix
+    }
     override fun onDestroy() {
-        Log.i(LOG_TAG, "onDestroy - Thread ID = " + Thread.currentThread().id)
         super.onDestroy()
+        Logger.i(LOG_TAG, "onDestroy - Canceling service scope")
+        serviceScope.cancel() // 取消所有与服务相关的协程
     }
 
     override fun onUnbind(intent: Intent?): Boolean {
-        Log.i(LOG_TAG, "onUnbind - Thread ID = " + Thread.currentThread().id)
+        Logger.i(LOG_TAG, "onUnbind - Thread ID = " + Thread.currentThread().id)
         stopSelf()
         return super.onUnbind(intent)
     }
+
     fun isConnected(): Boolean {
         if (!::network.isInitialized) return false
-        return network?.connect(1000)
+        return network.connect(1000)
     }
 
     fun stop() {
-        running=false
+        running = false
         stopSelf()
+        Openp2p.stop()
     }
+
     @RequiresApi(Build.VERSION_CODES.O)
     private fun createNotificationChannel(channelId: String, channelName: String): String? {
         val chan = NotificationChannel(
@@ -112,4 +398,55 @@ class OpenP2PService : Service() {
         service.createNotificationChannel(chan)
         return channelId
     }
+}
+
+// 获取本机所有IP地址和对应的子网信息
+fun getLocalIpAndSubnet(): List<Pair<String, Int>> {
+    val localIps = mutableListOf<Pair<String, Int>>()
+    val networkInterfaces = NetworkInterface.getNetworkInterfaces()
+    // 手动添加测试数据
+    //localIps.add(Pair("192.168.3.33", 24))
+    while (networkInterfaces.hasMoreElements()) {
+        val networkInterface = networkInterfaces.nextElement()
+        if (networkInterface.isUp && !networkInterface.isLoopback) {
+            val interfaceAddresses = networkInterface.interfaceAddresses
+            for (interfaceAddress in interfaceAddresses) {
+                val address = interfaceAddress.address
+                val prefixLength = interfaceAddress.networkPrefixLength
+                if (address is InetAddress) {
+                    localIps.add(Pair(address.hostAddress, prefixLength.toInt()))
+                }
+            }
+        }
+    }
+    return localIps
+}
+
+// 判断某个IP是否与本机某网段匹配
+fun isSameSubnet(ipAddress: String, localIps: List<Pair<String, Int>>): Boolean {
+    val targetIp = InetAddress.getByName(ipAddress).address
+    for ((localIp, prefixLength) in localIps) {
+        val localIpBytes = InetAddress.getByName(localIp).address
+        val mask = createSubnetMask(prefixLength, localIpBytes.size) // 动态生成掩码
+
+        // 比较目标 IP 和本地 IP 的网络部分
+        if (targetIp.indices.all { i ->
+                (targetIp[i].toInt() and mask[i].toInt()) == (localIpBytes[i].toInt() and mask[i].toInt())
+            }) {
+            return true
+        }
+    }
+    return false
+}
+
+// 根据前缀长度动态生成子网掩码
+fun createSubnetMask(prefixLength: Int, addressLength: Int): ByteArray {
+    val mask = ByteArray(addressLength)
+    for (i in 0 until prefixLength / 8) {
+        mask[i] = 0xFF.toByte()
+    }
+    if (prefixLength % 8 != 0) {
+        mask[prefixLength / 8] = (0xFF shl (8 - (prefixLength % 8))).toByte()
+    }
+    return mask
 }

app/app/src/main/java/cn/openp2p/log.kt

@@ -0,0 +1,91 @@
+package cn.openp2p
+import android.util.Log
+import java.text.SimpleDateFormat
+import java.io.BufferedWriter
+import java.io.File
+import java.io.FileWriter
+import java.io.IOException
+import java.util.Date
+import java.util.Locale
+
+object Logger {
+    private const val LOG_TAG = "OpenP2PLogger"
+    private var logFile: File? = null
+    private var bufferedWriter: BufferedWriter? = null
+
+    // 初始化日志文件
+    fun init(logDir: File, logFileName: String = "app.log") {
+        if (!logDir.exists()) logDir.mkdirs()
+        logFile = File(logDir, logFileName)
+
+        try {
+            bufferedWriter = BufferedWriter(FileWriter(logFile, true))
+        } catch (e: IOException) {
+            Log.e(LOG_TAG, "Failed to initialize BufferedWriter: ${e.message}")
+        }
+    }
+
+    // 写日志（线程安全）
+    @Synchronized
+    fun log(level: String, tag: String, message: String, throwable: Throwable? = null) {
+        val timestamp = SimpleDateFormat("yyyy-MM-dd HH:mm:ss", Locale.getDefault()).format(Date())
+        val logMessage = "$timestamp $level $tag: $message"
+
+        // 打印到 console
+        when (level) {
+            "ERROR" -> Log.e(tag, message, throwable)
+            "WARN" -> Log.w(tag, message, throwable)
+            "INFO" -> Log.i(tag, message)
+            "DEBUG" -> Log.d(tag, message)
+            "VERBOSE" -> Log.v(tag, message)
+        }
+
+        // 写入文件
+        try {
+            bufferedWriter?.apply {
+                write(logMessage)
+                newLine()
+                flush()
+            }
+            throwable?.let {
+                bufferedWriter?.apply {
+                    write(Log.getStackTraceString(it))
+                    newLine()
+                    flush()
+                }
+            }
+        } catch (e: IOException) {
+            Log.e(LOG_TAG, "Failed to write log to file: ${e.message}")
+        }
+    }
+
+    // 清理资源
+    fun close() {
+        try {
+            bufferedWriter?.close()
+        } catch (e: IOException) {
+            Log.e(LOG_TAG, "Failed to close BufferedWriter: ${e.message}")
+        }
+    }
+
+    // 简化方法
+    fun e(tag: String, message: String, throwable: Throwable? = null) {
+        log("ERROR", tag, message, throwable)
+    }
+
+    fun w(tag: String, message: String, throwable: Throwable? = null) {
+        log("WARN", tag, message, throwable)
+    }
+
+    fun i(tag: String, message: String) {
+        log("INFO", tag, message)
+    }
+
+    fun d(tag: String, message: String) {
+        log("DEBUG", tag, message)
+    }
+
+    fun v(tag: String, message: String) {
+        log("VERBOSE", tag, message)
+    }
+}

app/app/src/main/java/cn/openp2p/ui/login/LoginActivity.kt

@@ -5,6 +5,7 @@ import android.app.Activity
 import android.app.ActivityManager
 import android.app.Notification
 import android.app.PendingIntent
+import android.content.BroadcastReceiver
 import android.content.ComponentName
 import android.content.Context
 import android.content.Intent
@@ -25,6 +26,7 @@ import androidx.annotation.StringRes
 import androidx.appcompat.app.AppCompatActivity
 import androidx.lifecycle.Observer
 import androidx.lifecycle.ViewModelProvider
+import cn.openp2p.Logger
 import cn.openp2p.OpenP2PService
 import cn.openp2p.R
 import cn.openp2p.databinding.ActivityLoginBinding
@@ -51,6 +53,14 @@ class LoginActivity : AppCompatActivity() {
     private lateinit var loginViewModel: LoginViewModel
     private lateinit var binding: ActivityLoginBinding
     private lateinit var mService: OpenP2PService
+    @RequiresApi(Build.VERSION_CODES.O)
+    override fun onActivityResult(requestCode: Int, resultCode: Int, data: Intent?) {
+        super.onActivityResult(requestCode, resultCode, data)
+        if (requestCode == 0 && resultCode == Activity.RESULT_OK) {
+            startService(Intent(this, OpenP2PService::class.java))
+        }
+    }
+
     @RequiresApi(Build.VERSION_CODES.O)
     override fun onCreate(savedInstanceState: Bundle?) {
         super.onCreate(savedInstanceState)
@@ -78,22 +88,18 @@ class LoginActivity : AppCompatActivity() {
                 token.error = getString(loginState.passwordError)
             }
         })
-        val intent1 = VpnService.prepare(this) ?: return
+        openp2pLog.setText(R.string.phone_setting)
-        loginViewModel.loginResult.observe(this@LoginActivity, Observer {
+        val intent = VpnService.prepare(this)
-            val loginResult = it ?: return@Observer
+         if (intent != null)
+         {
+             Log.i("openp2p", "VpnService.prepare need permission");
+             startActivityForResult(intent, 0)
+         }
+         else {
+             Log.i("openp2p", "VpnService.prepare ready");
+             onActivityResult(0, Activity.RESULT_OK, null)
+         }
 
-            loading.visibility = View.GONE
-            if (loginResult.error != null) {
-                showLoginFailed(loginResult.error)
-            }
-            if (loginResult.success != null) {
-                updateUiWithUser(loginResult.success)
-            }
-            setResult(Activity.RESULT_OK)
-
-            //Complete and destroy login activity once successful
-            finish()
-        })
 
         profile.setOnClickListener {
             val url = "https://console.openp2p.cn/profile"
@@ -110,17 +116,23 @@ class LoginActivity : AppCompatActivity() {
             }
 
             openp2pLog.setText(R.string.phone_setting)
-            token.setText(Openp2p.getToken(getExternalFilesDir(null).toString()))
+
             login.setOnClickListener {
                 if (login.text.toString()=="退出"){
-//                    val intent = Intent(this@LoginActivity, OpenP2PService::class.java)
+                //    val intent = Intent(this@LoginActivity, OpenP2PService::class.java)
-//                    stopService(intent)
+                //    stopService(intent)
                     Log.i(LOG_TAG, "quit")
                     mService.stop()
-                    unbindService(connection)
+
                     val intent = Intent(this@LoginActivity, OpenP2PService::class.java)
                     stopService(intent)
+                    // 解绑服务
+                    unbindService(connection)
+
+                    // 结束当前 Activity
+                    finish() // 或者使用 finishAffinity() 来结束整个应用程序
                     exitAPP()
+                    // finishAffinity()
 
                 }
                 login.setText("退出")
@@ -139,13 +151,20 @@ class LoginActivity : AppCompatActivity() {
                             if (isConnect) {
                                 onlineState.setText("在线")
                             } else {
-                                onlineState.setText("离线")
+                                onlineState.setText("正在登录")
                             }
                         }
                     } while (true)
                 }
 
             }
+            val tokenText = Openp2p.getToken(getExternalFilesDir(null).toString())
+            token.setText(tokenText.toString())
+            // Check token length and automatically click login if length > 10
+            if (tokenText.length > 10) {
+//                Logger.log("performClick ")
+                login.performClick()
+            }
         }
     }
     @RequiresApi(Build.VERSION_CODES.LOLLIPOP)

app/app/src/main/res/layout/activity_login.xml

@@ -13,28 +13,27 @@
 
     <EditText
         android:id="@+id/token"
-        android:layout_width="225dp"
+        android:layout_width="250dp"
-        android:layout_height="46dp"
+        android:layout_height="45dp"
         android:hint="Token"
         android:imeActionLabel="@string/action_sign_in_short"
         android:imeOptions="actionDone"
         android:selectAllOnFocus="true"
         app:layout_constraintStart_toStartOf="parent"
-        app:layout_constraintTop_toTopOf="parent"
+        app:layout_constraintTop_toTopOf="parent" />
-         />
 
     <Button
         android:id="@+id/login"
-        android:layout_width="wrap_content"
+        android:layout_width="85dp"
-        android:layout_height="wrap_content"
+        android:layout_height="45dp"
         android:layout_gravity="start"
         android:layout_marginStart="24dp"
         android:layout_marginLeft="24dp"
         android:enabled="false"
         android:text="@string/action_sign_in"
+        app:layout_constraintEnd_toEndOf="parent"
         app:layout_constraintStart_toEndOf="@+id/token"
-        app:layout_constraintTop_toTopOf="parent"
+        app:layout_constraintTop_toTopOf="parent" />
-        tools:layout_editor_absoluteY="-2dp" />
 
 
 
@@ -58,28 +57,36 @@
         android:id="@+id/openp2pLog"
         android:layout_width="359dp"
         android:layout_height="548dp"
+        android:layout_marginTop="24dp"
         android:ems="10"
-        android:inputType="none"
-        android:textIsSelectable="true"
         android:focusable="false"
+        android:inputType="none"
         android:text="Name"
+        android:textIsSelectable="true"
+        app:layout_constraintEnd_toEndOf="parent"
         app:layout_constraintStart_toStartOf="parent"
         app:layout_constraintTop_toBottomOf="@+id/onlineState" />
+
     <Button
         android:id="@+id/profile"
-        android:layout_width="wrap_content"
+        android:layout_width="250dp"
-        android:layout_height="wrap_content"
+        android:layout_height="45dp"
+        android:layout_marginTop="8dp"
         android:text="打开控制台查看Token"
         app:layout_constraintStart_toStartOf="parent"
         app:layout_constraintTop_toBottomOf="@+id/token" />
 
     <EditText
         android:id="@+id/onlineState"
-        android:layout_width="113dp"
+        android:layout_width="85dp"
         android:layout_height="45dp"
+        android:layout_marginStart="24dp"
+        android:layout_marginLeft="24dp"
+        android:layout_marginTop="8dp"
         android:ems="10"
         android:inputType="textPersonName"
         android:text="未登录"
+        app:layout_constraintEnd_toEndOf="parent"
         app:layout_constraintStart_toEndOf="@+id/profile"
         app:layout_constraintTop_toBottomOf="@+id/login" />
 

app/app/src/main/res/values/strings.xml

@@ -10,7 +10,14 @@
     <string name="invalid_password">Token可以在 https://console.openp2p.cn/profile 获得</string>
     <string name="login_failed">"Login failed"</string>
     <string name="phone_setting">"安卓系统默认设置的”杀后台进程“会导致 OpenP2P 在后台运行一会后，被系统杀死进程，导致您的体验受到影响。您可以通过以下方式修改几个设置，解决此问题：
-
+华为鸿蒙：
+    1. 允许应用后台运行：进入设置 → 搜索进入 应用启动管理 → 关闭 OpenP2P 的 自动管理 开关 → 在弹框中勾选 允许后台活动 
+    2. 避免应用被电池优化程序清理：进入设置 → 搜索进入电池优化 → 不允许 →选择所有应用 → 找到无法后台运行的应用 → 设置为不允许
+    3. 关闭省电模式：进入设置 → 电池 → 关闭 省电模式 开关
+    4. 保持设备网络连接：进入设置  → 电池 → 更多电池设置 → 开启 休眠时始终保持网络连接 开关。
+    5. 给后台运行的应用加锁：打开应用后 → 进入多任务界面 → 下拉选中的卡片进行加锁 → 然后点击清理图标清理其他不经常使用的应用
+    6. 设置开发人员选项中相关开关：进入设置 → 搜索进入 开发人员选项 → 找到 不保留活动 开关后关闭 → 并在 后台进程限制 选择 标准限制
+    
 华为手机：
     进入”设置“，搜索并进入“电池优化“界面，选中 OpenP2P 程序，不允许系统对其进行电池优化；
     进入”设置“，进入”应用管理“界面，选中 OpenP2P 程序，点击”耗电情况“，开启”允许后台活动“即可；

app/build.gradle

@@ -0,0 +1,38 @@
+// Top-level build file where you can add configuration options common to all sub-projects/modules.
+
+buildscript {
+
+    ext.kotlin_version = "1.8.20"
+    repositories {
+        google()
+        mavenCentral()
+    }
+    dependencies {
+        classpath "com.android.tools.build:gradle:8.1.3"
+        classpath "org.jetbrains.kotlin:kotlin-gradle-plugin:$kotlin_version"
+
+        // NOTE: Do not place your application dependencies here; they belong
+        // in the individual module build.gradle files
+    }
+}
+
+
+allprojects {
+    repositories {
+        google()
+        mavenCentral()
+        jcenter() // Warning: this repository is going to shut down soon
+    }
+}
+allprojects {
+    repositories {
+        maven { url 'https://maven.aliyun.com/repository/google' }
+        maven { url 'https://maven.aliyun.com/repository/central' }
+        maven { url 'https://maven.aliyun.com/repository/public' }
+        maven { url 'https://jitpack.io' }
+    }
+}
+
+task clean(type: Delete) {
+    delete rootProject.buildDir
+}

app/gradle.properties

@@ -6,7 +6,7 @@
 # http://www.gradle.org/docs/current/userguide/build_environment.html
 # Specifies the JVM arguments used for the daemon process.
 # The setting is particularly useful for tweaking memory settings.
-org.gradle.jvmargs=-Xmx2048m -Dfile.encoding=UTF-8
+org.gradle.jvmargs=-Xmx4096m -Dfile.encoding=UTF-8
 # When configured, Gradle will run in incubating parallel mode.
 # This option should only be used with decoupled projects. More details, visit
 # http://www.gradle.org/docs/current/userguide/multi_project_builds.html#sec:decoupled_projects
@@ -16,4 +16,6 @@ org.gradle.jvmargs=-Xmx2048m -Dfile.encoding=UTF-8
 # https://developer.android.com/topic/libraries/support-library/androidx-rn
 android.useAndroidX=true
 # Kotlin code style for this project: "official" or "obsolete":
-kotlin.code.style=official
+kotlin.code.style=official
+org.gradle.caching=true
+

app/gradle/wrapper/gradle-wrapper.properties

@@ -1,6 +1,6 @@
-#Sat Oct 22 21:46:24 CST 2022
+#Tue Dec 05 16:04:08 CST 2023
 distributionBase=GRADLE_USER_HOME
-distributionUrl=https\://services.gradle.org/distributions/gradle-6.7.1-bin.zip
 distributionPath=wrapper/dists
-zipStorePath=wrapper/dists
+distributionUrl=https\://services.gradle.org/distributions/gradle-8.2-bin.zip
 zipStoreBase=GRADLE_USER_HOME
+zipStorePath=wrapper/dists

cmd/openp2p.go

@@ -1,7 +1,9 @@
-package main
+package main
-
+
-import openp2p "openp2p/core"
+import (
-
+	op2p "openp2p/core"
-func main() {
+)
-	openp2p.Run()
+
-}
+func main() {
+	op2p.Run()
+}

core/bandwidthLimit.go

@@ -1,45 +0,0 @@
-package openp2p
-
-import (
-	"sync"
-	"time"
-)
-
-// BandwidthLimiter ...
-type BandwidthLimiter struct {
-	ts           time.Time
-	bw           int // mbps
-	freeBytes    int // bytes
-	maxFreeBytes int // bytes
-	mtx          sync.Mutex
-}
-
-// mbps
-func newBandwidthLimiter(bw int) *BandwidthLimiter {
-	return &BandwidthLimiter{
-		bw:           bw,
-		ts:           time.Now(),
-		maxFreeBytes: bw * 1024 * 1024 / 8,
-		freeBytes:    bw * 1024 * 1024 / 8,
-	}
-}
-
-// Add ...
-func (bl *BandwidthLimiter) Add(bytes int) {
-	if bl.bw <= 0 {
-		return
-	}
-	bl.mtx.Lock()
-	defer bl.mtx.Unlock()
-	// calc free flow 1000*1000/1024/1024=0.954; 1024*1024/1000/1000=1.048
-	bl.freeBytes += int(time.Since(bl.ts) * time.Duration(bl.bw) / 8 / 954)
-	if bl.freeBytes > bl.maxFreeBytes {
-		bl.freeBytes = bl.maxFreeBytes
-	}
-	bl.freeBytes -= bytes
-	bl.ts = time.Now()
-	if bl.freeBytes < 0 {
-		// sleep for the overflow
-		time.Sleep(time.Millisecond * time.Duration(-bl.freeBytes/(bl.bw*1048/8)))
-	}
-}

core/common.go

@@ -2,16 +2,22 @@ package openp2p
 
 import (
 	"bytes"
+	"context"
 	"crypto/aes"
 	"crypto/cipher"
 	"crypto/tls"
+	"encoding/binary"
 	"encoding/json"
 	"fmt"
+	"io"
+	"math"
+	"math/big"
 	"math/rand"
 	"net"
 	"net/http"
 	"os"
 	"os/exec"
+	"runtime"
 	"strconv"
 	"strings"
 	"time"
@@ -20,7 +26,6 @@ import (
 const MinNodeNameLen = 8
 
 func getmac(ip string) string {
-	//get mac relative to the ip address which connected to the mq.
 	ifaces, err := net.Interfaces()
 	if err != nil {
 		return ""
@@ -127,20 +132,19 @@ func netInfo() *NetInfo {
 		client := &http.Client{Transport: tr, Timeout: time.Second * 10}
 		r, err := client.Get("https://ifconfig.co/json")
 		if err != nil {
-			gLog.Println(LvDEBUG, "netInfo error:", err)
+			gLog.d("netInfo error:%s", err)
 			continue
 		}
 		defer r.Body.Close()
 		buf := make([]byte, 1024*64)
 		n, err := r.Body.Read(buf)
-		if err != nil {
+		if err != nil && err != io.EOF {
-			gLog.Println(LvDEBUG, "netInfo error:", err)
+			gLog.d("error reading response body: %s", err)
 			continue
 		}
 		rsp := NetInfo{}
-		err = json.Unmarshal(buf[:n], &rsp)
+		if err = json.Unmarshal(buf[:n], &rsp); err != nil {
-		if err != nil {
+			gLog.d("wrong NetInfo:%s", err)
-			gLog.Printf(LvERROR, "wrong NetInfo:%s", err)
 			continue
 		}
 		return &rsp
@@ -199,8 +203,12 @@ func parseMajorVer(ver string) int {
 	return 0
 }
 
-func IsIPv6(address string) bool {
+func IsIPv6(ipStr string) bool {
-	return strings.Count(address, ":") >= 2
+	ip := net.ParseIP(ipStr)
+	if ip == nil {
+		return false
+	}
+	return ip.To16() != nil && ip.To4() == nil
 }
 
 var letters = []byte("abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ1234567890-")
@@ -212,3 +220,186 @@ func randStr(n int) string {
 	}
 	return string(b)
 }
+
+func execCommand(commandPath string, wait bool, arg ...string) (err error) {
+	command := exec.Command(commandPath, arg...)
+	err = command.Start()
+	if err != nil {
+		return
+	}
+	if wait {
+		err = command.Wait()
+	}
+	return
+}
+
+func sanitizeFileName(fileName string) string {
+	validFileName := fileName
+	invalidChars := []string{"\\", "/", ":", "*", "?", "\"", "<", ">", "|"}
+	for _, char := range invalidChars {
+		validFileName = strings.ReplaceAll(validFileName, char, " ")
+	}
+	return validFileName
+}
+
+func prettyJson(s interface{}) string {
+	jsonData, err := json.MarshalIndent(s, "", "  ")
+	if err != nil {
+		fmt.Println("Error marshalling JSON:", err)
+		return ""
+	}
+	return string(jsonData)
+}
+
+func inetAtoN(ipstr string) (uint32, error) { // support both ipnet or single ip
+	i, _, err := net.ParseCIDR(ipstr)
+	if err != nil {
+		i = net.ParseIP(ipstr)
+		if i == nil {
+			return 0, err
+		}
+	}
+	ret := big.NewInt(0)
+	ret.SetBytes(i.To4())
+	return uint32(ret.Int64()), nil
+}
+
+func calculateChecksum(data []byte) uint16 {
+	length := len(data)
+	sum := uint32(0)
+
+	// Calculate the sum of 16-bit words
+	for i := 0; i < length-1; i += 2 {
+		sum += uint32(binary.BigEndian.Uint16(data[i : i+2]))
+	}
+
+	// Add the last byte (if odd length)
+	if length%2 != 0 {
+		sum += uint32(data[length-1])
+	}
+
+	// Fold 32-bit sum to 16 bits
+	sum = (sum >> 16) + (sum & 0xffff)
+	sum += (sum >> 16)
+
+	return uint16(^sum)
+}
+
+func min(nums ...int32) int32 {
+	if len(nums) == 0 {
+		return 0 // 如果没有输入，返回最大值
+	}
+
+	minVal := nums[0]
+	for _, num := range nums[1:] {
+		if num < minVal {
+			minVal = num
+		}
+	}
+	return minVal
+}
+
+func calcRetryTimeRelay(x float64) float64 {
+	return 10 + math.Exp(0.8*(x-3.6))
+}
+func calcRetryTimeDirect(x float64) float64 {
+	return 10 + math.Exp(2.8*(x-4))
+}
+
+func isAndroid() bool {
+	if runtime.GOOS == "android" {
+		return true
+	}
+	data, err := os.ReadFile("/proc/version")
+	if err != nil {
+		return false
+	}
+	return strings.Contains(string(data), "Android")
+}
+
+func moveFile(src, dst string) error {
+	err := os.Rename(src, dst)
+	if err == nil {
+		return nil
+	}
+	// windows could not rename running executable, so copy then delete
+	if runtime.GOOS == "windows" {
+		err = copyFile(src, dst)
+		if err != nil {
+			return err
+		}
+
+		os.Remove(src)
+	}
+	return nil
+}
+
+func copyFile(src, dst string) error {
+	sourceFile, err := os.Open(src)
+	if err != nil {
+		return err
+	}
+	defer sourceFile.Close()
+
+	destFile, err := os.Create(dst)
+	if err != nil {
+		return err
+	}
+	defer destFile.Close()
+
+	_, err = io.Copy(destFile, sourceFile)
+	if err != nil {
+		return err
+	}
+
+	return destFile.Sync()
+}
+
+func resolveServerIP(host string) ([]string, error) {
+	ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
+	defer cancel()
+
+	// 先系统 DNS
+	ips, err := net.DefaultResolver.LookupHost(ctx, host)
+	if err == nil && len(ips) > 0 {
+		gLog.i("system dns resolved %s -> %v", host, ips)
+		return ips, nil
+	}
+
+	gLog.e("system dns resolve failed for %s: %v", host, err)
+	gLog.i("retry with fallback dns...")
+
+	// 再 fallback dns
+	return lookupWithCustomDNS(ctx, host)
+}
+func lookupWithCustomDNS(ctx context.Context, domain string) ([]string, error) {
+	resolver := &net.Resolver{
+		PreferGo: true,
+		Dial: func(ctx context.Context, network, address string) (net.Conn, error) {
+			dialer := &net.Dialer{Timeout: 5 * time.Second}
+
+			// 先 119.29.29.29
+			conn, err := dialer.DialContext(ctx, network, "119.29.29.29:53")
+			if err == nil {
+				return conn, nil
+			}
+
+			// 再 8.8.8.8
+			return dialer.DialContext(ctx, network, "8.8.8.8:53")
+		},
+	}
+
+	return resolver.LookupHost(ctx, domain)
+}
+
+func writeFull(w io.Writer, data []byte) error {
+	totalWritten := 0
+	for totalWritten < len(data) {
+		n, err := w.Write(data[totalWritten:])
+		if err != nil {
+			return fmt.Errorf("write failed after %d bytes: %w", totalWritten, err)
+		}
+		totalWritten += n
+	}
+	return nil
+}

core/common_test.go

@@ -1,6 +1,7 @@
 package openp2p
 
 import (
+	"fmt"
 	"log"
 	"testing"
 )
@@ -94,3 +95,91 @@ func TestParseMajorVer(t *testing.T) {
 	assertParseMajorVer(t, "3.0.0", 3)
 
 }
+
+func TestIsIPv6(t *testing.T) {
+	tests := []struct {
+		ipStr string
+		want  bool
+	}{
+		{"2001:0db8:85a3:0000:0000:8a2e:0370:7334", true}, // 有效的 IPv6 地址
+		{"2001:db8::2:1", true},                           // 有效的 IPv6 地址
+		{"192.168.1.1", false},                            // 无效的 IPv6 地址，是 IPv4
+		{"2001:db8::G:1", false},                          // 无效的 IPv6 地址，包含非法字符
+		// 可以添加更多测试用例
+	}
+
+	for _, tt := range tests {
+		got := IsIPv6(tt.ipStr)
+		if got != tt.want {
+			t.Errorf("isValidIPv6(%s) = %v, want %v", tt.ipStr, got, tt.want)
+		}
+	}
+}
+
+func TestNodeID(t *testing.T) {
+	node1 := "n1-stable"
+	node2 := "tony-stable"
+	nodeID1 := NodeNameToID(node1)
+	nodeID2 := NodeNameToID(node2)
+	if nodeID1 < nodeID2 {
+		fmt.Printf("%s < %s\n", node1, node2)
+	} else {
+		fmt.Printf("%s >= %s\n", node1, node2)
+	}
+}
+
+func TestCalcRetryTime(t *testing.T) {
+	// 0-2 < 13s
+	// 3-5:300
+	// 6-10:600
+	tests := []struct {
+		retryNum float64
+		want     float64
+	}{
+		{1.0, 10},
+		{5.0, 13},
+		{10.0, 180},
+		{15.0, 9000},
+		{18.0, 90000},
+		// 可以添加更多测试用例
+	}
+
+	for _, tt := range tests {
+		got := calcRetryTimeRelay(tt.retryNum)
+		if got < tt.want*0.85 || got > tt.want*1.15 {
+			t.Errorf("calcRetryTime(%f) = %f, want %f", tt.retryNum, got, tt.want)
+		}
+	}
+
+	for i := 0; i < 20; i++ {
+		log.Printf("%d retryTime=%fs", i, calcRetryTimeRelay(float64(i)))
+	}
+}
+
+func TestCalcRetryTimeDirect(t *testing.T) {
+	// 0-2 < 13s
+	// 3-5:300
+	// 6-10:600
+	tests := []struct {
+		retryNum float64
+		want     float64
+	}{
+		{1.0, 10},
+		{5.0, 13},
+		{10.0, 180},
+		{15.0, 9000},
+		{18.0, 90000},
+		// 可以添加更多测试用例
+	}
+
+	for _, tt := range tests {
+		got := calcRetryTimeRelay(tt.retryNum)
+		if got < tt.want*0.85 || got > tt.want*1.15 {
+			t.Errorf("calcRetryTime(%f) = %f, want %f", tt.retryNum, got, tt.want)
+		}
+	}
+
+	for i := 0; i < 20; i++ {
+		log.Printf("%d retryTime=%fs", i, calcRetryTimeDirect(float64(i)))
+	}
+}

core/config.go

@@ -3,8 +3,10 @@ package openp2p
 import (
 	"encoding/json"
 	"flag"
-	"io/ioutil"
+	"fmt"
 	"os"
+	"strconv"
+	"strings"
 	"sync"
 	"time"
 )
@@ -13,18 +15,25 @@ var gConf Config
 
 type AppConfig struct {
 	// required
-	AppName  string
+	AppName          string
-	Protocol string
+	Protocol         string
-	SrcPort  int
+	UnderlayProtocol string
-	PeerNode string
+	PunchPriority    int // bitwise DisableTCP|DisableUDP|TCPFirst  0:tcp and udp both enable, udp first
-	DstPort  int
+	Whitelist        string
-	DstHost  string
+	SrcPort          int
-	PeerUser string
+	PeerNode         string
-	Enabled  int // default:1
+	DstPort          int
+	DstHost          string
+	PeerUser         string
+	RelayNode        string
+	ForceRelay       int // default:0 disable;1 enable
+	Enabled          int // default:1
 	// runtime info
+	relayMode        string // private|public
 	peerVersion      string
 	peerToken        uint64
 	peerNatType      int
+	peerLanIP        string
 	hasIPv4          int
 	peerIPv6         string
 	hasUPNPorNATPMP  int
@@ -38,16 +47,117 @@ type AppConfig struct {
 	connectTime      time.Time
 	fromToken        uint64
 	linkMode         string
-	isUnderlayServer int // TODO: bool?
+	isUnderlayServer int
+}
+
+const (
+	PunchPriorityUDPFirst = 0
+	PunchPriorityTCPFirst = 1
+	PunchPriorityTCPOnly  = 1 << 1
+	PunchPriorityUDPOnly  = 1 << 2
+)
+
+func (c *AppConfig) ID() uint64 {
+	if c.SrcPort == 0 { // memapp
+		return NodeNameToID(c.PeerNode)
+	}
+	if c.Protocol == "tcp" {
+		return uint64(c.SrcPort) * 10
+	}
+	return uint64(c.SrcPort)*10 + 1
+}
+
+func (c *AppConfig) LogPeerNode() string {
+	if c.relayMode == "public" { // memapp
+		return fmt.Sprintf("%d", NodeNameToID(c.PeerNode))
+	}
+	return c.PeerNode
 }
 
-// TODO: add loglevel, maxlogfilesize
 type Config struct {
-	Network    NetworkConfig `json:"network"`
+	Network NetworkConfig `json:"network"`
-	Apps       []*AppConfig  `json:"apps"`
+	Apps    []*AppConfig  `json:"apps"`
-	LogLevel   int
+
-	daemonMode bool
+	LogLevel              int
-	mtx        sync.Mutex
+	MaxLogSize            int
+	TLSInsecureSkipVerify bool
+	Forcev6               bool
+	daemonMode            bool
+	mtx                   sync.RWMutex
+	fileMtx               sync.Mutex
+	sdwanMtx              sync.Mutex
+	sdwan                 SDWANInfo
+	delNodes              []*SDWANNode
+	addNodes              []*SDWANNode
+}
+
+func (c *Config) getSDWAN() SDWANInfo {
+	c.sdwanMtx.Lock()
+	defer c.sdwanMtx.Unlock()
+	return c.sdwan
+}
+
+func (c *Config) getDelNodes() []*SDWANNode {
+	c.sdwanMtx.Lock()
+	defer c.sdwanMtx.Unlock()
+	return c.delNodes
+}
+
+func (c *Config) getAddNodes() []*SDWANNode {
+	c.sdwanMtx.Lock()
+	defer c.sdwanMtx.Unlock()
+	return c.addNodes
+}
+
+func (c *Config) resetSDWAN() {
+	c.sdwanMtx.Lock()
+	defer c.sdwanMtx.Unlock()
+	c.delNodes = []*SDWANNode{}
+	c.addNodes = []*SDWANNode{}
+	c.sdwan = SDWANInfo{}
+}
+func (c *Config) setSDWAN(s SDWANInfo) {
+	c.sdwanMtx.Lock()
+	defer c.sdwanMtx.Unlock()
+	allNew := false
+	if c.sdwan.GetResourceByNodeName(c.Network.Node) != s.GetResourceByNodeName(c.Network.Node) {
+		allNew = true
+	}
+	// get old-new
+	c.delNodes = []*SDWANNode{}
+	for _, oldNode := range c.sdwan.Nodes {
+		isDeleted := true
+		for _, newNode := range s.Nodes {
+			if oldNode.Name == newNode.Name && oldNode.IP == newNode.IP && oldNode.Resource == newNode.Resource && c.sdwan.Mode == s.Mode && c.sdwan.CentralNode == s.CentralNode {
+				isDeleted = false
+				break
+			}
+		}
+		if isDeleted || allNew {
+			c.delNodes = append(c.delNodes, oldNode)
+		}
+	}
+	// get new-old
+	c.addNodes = []*SDWANNode{}
+	for _, newNode := range s.Nodes {
+		isNew := true
+		for _, oldNode := range c.sdwan.Nodes {
+			if oldNode.Name == newNode.Name && oldNode.IP == newNode.IP && oldNode.Resource == newNode.Resource && c.sdwan.Mode == s.Mode && c.sdwan.CentralNode == s.CentralNode {
+				isNew = false
+				break
+			}
+		}
+		if isNew || allNew {
+			c.addNodes = append(c.addNodes, newNode)
+		}
+	}
+	c.sdwan = s
+	if c.sdwan.TunnelNum < 2 {
+		c.sdwan.TunnelNum = 2 // DEBUG
+	}
+	// if c.sdwan.TunnelNum > 3 {
+	// 	c.sdwan.TunnelNum = 3
+	// }
 }
 
 func (c *Config) switchApp(app AppConfig, enabled int) {
@@ -58,55 +168,130 @@ func (c *Config) switchApp(app AppConfig, enabled int) {
 			c.Apps[i].Enabled = enabled
 			c.Apps[i].retryNum = 0
 			c.Apps[i].nextRetryTime = time.Now()
-			return
+			break
 		}
 	}
+	c.save()
+}
+
+// TODO: move to p2pnetwork
+func (c *Config) retryApp(peerNode string) {
+	GNetwork.apps.Range(func(id, i interface{}) bool {
+		app := i.(*p2pApp)
+		if app.config.PeerNode == peerNode {
+			app.Retry(true)
+		}
+		if app.config.RelayNode == peerNode {
+			app.Retry(false)
+			gLog.d("retry app relay=%s", app.config.LogPeerNode())
+		}
+		return true
+	})
+}
+
+func (c *Config) retryAllApp() {
+	GNetwork.apps.Range(func(id, i interface{}) bool {
+		app := i.(*p2pApp)
+		app.Retry(true)
+		return true
+	})
+}
+
+func (c *Config) retryAllMemApp() {
+	GNetwork.apps.Range(func(id, i interface{}) bool {
+		app := i.(*p2pApp)
+		if app.config.SrcPort != 0 {
+			return true
+		}
+		if app.tunnelNum != int(gConf.sdwan.TunnelNum) {
+			gLog.d("memapp %s tunnelNum changed from %d to %d, delete it and not retry", app.config.LogPeerNode(), app.tunnelNum, gConf.sdwan.TunnelNum)
+			GNetwork.DeleteApp(app.config)
+			return true
+		}
+		app.Retry(true)
+		return true
+	})
 }
 
 func (c *Config) add(app AppConfig, override bool) {
+	if app.AppName == "" {
+		app.AppName = fmt.Sprintf("%d", app.ID())
+	}
 	c.mtx.Lock()
 	defer c.mtx.Unlock()
-	if app.SrcPort == 0 || app.DstPort == 0 {
-		gLog.Println(LvERROR, "invalid app ", app)
-		return
-	}
 	if override {
 		for i := 0; i < len(c.Apps); i++ {
-			if c.Apps[i].Protocol == app.Protocol && c.Apps[i].SrcPort == app.SrcPort {
+			if c.Apps[i].PeerNode == app.PeerNode && c.Apps[i].Protocol == app.Protocol && c.Apps[i].SrcPort == app.SrcPort {
 				c.Apps[i] = &app // override it
 				return
 			}
 		}
 	}
 	c.Apps = append(c.Apps, &app)
+	if app.SrcPort != 0 {
+		c.save()
+	}
 }
 
 func (c *Config) delete(app AppConfig) {
-	if app.SrcPort == 0 || app.DstPort == 0 {
-		return
-	}
 	c.mtx.Lock()
 	defer c.mtx.Unlock()
 	for i := 0; i < len(c.Apps); i++ {
-		if c.Apps[i].Protocol == app.Protocol && c.Apps[i].SrcPort == app.SrcPort {
+		if (app.SrcPort != 0 && c.Apps[i].Protocol == app.Protocol && c.Apps[i].SrcPort == app.SrcPort) || // normal app
-			c.Apps = append(c.Apps[:i], c.Apps[i+1:]...)
+			(app.SrcPort == 0 && c.Apps[i].SrcPort == 0 && c.Apps[i].PeerNode == app.PeerNode) { // memapp
-			return
+			if i == len(c.Apps)-1 {
+				c.Apps = c.Apps[:i]
+			} else {
+				c.Apps = append(c.Apps[:i], c.Apps[i+1:]...)
+			}
+			break
 		}
 	}
+	if app.SrcPort != 0 {
+		c.save()
+	}
 }
 
 func (c *Config) save() {
-	c.mtx.Lock()
+	c.fileMtx.Lock()
-	defer c.mtx.Unlock()
+	defer c.fileMtx.Unlock()
-	data, _ := json.MarshalIndent(c, "", "  ")
+	if c.Network.Token == 0 {
-	err := ioutil.WriteFile("config.json", data, 0644)
+		gLog.e("c.Network.Token == 0 skip save")
-	if err != nil {
+		return
-		gLog.Println(LvERROR, "save config.json error:", err)
 	}
+	data, err := json.MarshalIndent(c, "", "  ")
+	if err != nil || len(data) < 16 {
+		gLog.e("MarshalIndent config.json error:%v,  len=%d", err, len(data))
+		return
+	}
+	err = os.WriteFile("config.json0", data, 0644)
+	if err != nil {
+		gLog.e("save config.json error:%v", err)
+	}
+
+	// verify if the file is written correctly
+	data, err = os.ReadFile("config.json0")
+	if err != nil {
+		return
+	}
+	var tmpConfig Config
+	err = json.Unmarshal(data, &tmpConfig)
+	if err != nil {
+		gLog.e("parse config.json error:", err)
+		return
+	}
+	err = os.Rename("config.json0", "config.json")
+	if err != nil {
+		gLog.e("rename config file error:%v", err)
+	}
+
 }
 
+// -d run, then worker serverport always WsPort.
+// func init() {
 func init() {
-	gConf.LogLevel = 1
+	gConf.LogLevel = int(LvINFO)
+	gConf.MaxLogSize = 1024 * 1024
 	gConf.Network.ShareBandwidth = 10
 	gConf.Network.ServerHost = "api.openp2p.cn"
 	gConf.Network.ServerPort = WsPort
@@ -114,24 +299,37 @@ func init() {
 }
 
 func (c *Config) load() error {
-	c.mtx.Lock()
+	c.fileMtx.Lock()
-	defer c.mtx.Unlock()
+	defer c.fileMtx.Unlock()
-	data, err := ioutil.ReadFile("config.json")
+	data, err := os.ReadFile("config.json")
 	if err != nil {
-		// gLog.Println(LevelERROR, "read config.json error:", err)
 		return err
 	}
+	c.mtx.Lock()
+	defer c.mtx.Unlock()
 	err = json.Unmarshal(data, &c)
 	if err != nil {
-		gLog.Println(LvERROR, "parse config.json error:", err)
+		gLog.e("parse config.json error:", err)
+		return err
 	}
+	var filteredApps []*AppConfig // filter memapp
+	for _, app := range c.Apps {
+		if app.SrcPort != 0 {
+			filteredApps = append(filteredApps, app)
+		}
+	}
+	c.Apps = filteredApps
+	c.Network.natType = NATUnknown
 	return err
 }
 
+// deal with multi-thread r/w
 func (c *Config) setToken(token uint64) {
 	c.mtx.Lock()
 	defer c.mtx.Unlock()
-	c.Network.Token = token
+	if token != 0 {
+		c.Network.Token = token
+	}
 }
 func (c *Config) setUser(user string) {
 	c.mtx.Lock()
@@ -142,71 +340,129 @@ func (c *Config) setNode(node string) {
 	c.mtx.Lock()
 	defer c.mtx.Unlock()
 	c.Network.Node = node
+	c.Network.nodeID = NodeNameToID(c.Network.Node)
+}
+func (c *Config) setForcev6(force bool) {
+	c.mtx.Lock()
+	defer c.mtx.Unlock()
+	c.Forcev6 = force
+}
+func (c *Config) nodeID() uint64 {
+	c.mtx.Lock()
+	defer c.mtx.Unlock()
+	if c.Network.nodeID == 0 {
+		c.Network.nodeID = NodeNameToID(c.Network.Node)
+	}
+	return c.Network.nodeID
 }
 func (c *Config) setShareBandwidth(bw int) {
 	c.mtx.Lock()
 	defer c.mtx.Unlock()
+	defer c.save()
 	c.Network.ShareBandwidth = bw
 }
+func (c *Config) setIPv6(v6 string) {
+	c.mtx.Lock()
+	defer c.mtx.Unlock()
+	c.Network.publicIPv6 = v6
+}
+func (c *Config) IPv6() string {
+	c.mtx.Lock()
+	defer c.mtx.Unlock()
+	return c.Network.publicIPv6
+}
 
 type NetworkConfig struct {
 	// local info
 	Token           uint64
 	Node            string
+	nodeID          uint64
 	User            string
 	localIP         string
 	mac             string
 	os              string
 	publicIP        string
+	previousIP      string // for publicIP change detect
 	natType         int
 	hasIPv4         int
 	publicIPv6      string // must lowwer-case not save json
 	hasUPNPorNATPMP int
 	ShareBandwidth  int
 	// server info
-	ServerHost string
+	ServerHost     string
-	ServerPort int
+	ServerIP       string
-	UDPPort1   int
+	ServerPort     int
-	UDPPort2   int
+	natDetectPort1 int
-	TCPPort    int
+	natDetectPort2 int
+	PublicIPPort   int // both tcp and udp
+	specTunnel     int
 }
 
-func parseParams(subCommand string) {
+func parseParams(subCommand string, cmd string) {
 	fset := flag.NewFlagSet(subCommand, flag.ExitOnError)
+	installPath := fset.String("installpath", "", "custom install path")
 	serverHost := fset.String("serverhost", "api.openp2p.cn", "server host ")
+	insecure := fset.Bool("insecure", false, "not verify TLS certificate")
 	serverPort := fset.Int("serverport", WsPort, "server port ")
 	// serverHost := flag.String("serverhost", "127.0.0.1", "server host ") // for debug
 	token := fset.Uint64("token", 0, "token")
 	node := fset.String("node", "", "node name. 8-31 characters. if not set, it will be hostname")
 	peerNode := fset.String("peernode", "", "peer node name that you want to connect")
 	dstIP := fset.String("dstip", "127.0.0.1", "destination ip ")
+	whiteList := fset.String("whitelist", "", "whitelist for p2pApp ")
 	dstPort := fset.Int("dstport", 0, "destination port ")
 	srcPort := fset.Int("srcport", 0, "source port ")
-	tcpPort := fset.Int("tcpport", 0, "tcp port for upnp or publicip")
+	publicIPPort := fset.Int("publicipport", 0, "public ip port for upnp or publicip")
 	protocol := fset.String("protocol", "tcp", "tcp or udp")
+	underlayProtocol := fset.String("underlay_protocol", "quic", "quic or kcp")
+	punchPriority := fset.Int("punch_priority", 0, "bitwise DisableTCP|DisableUDP|UDPFirst  0:tcp and udp both enable, tcp first")
 	appName := fset.String("appname", "", "app name")
+	relayNode := fset.String("relaynode", "", "relaynode")
 	shareBandwidth := fset.Int("sharebandwidth", 10, "N mbps share bandwidth limit, private network no limit")
 	daemonMode := fset.Bool("d", false, "daemonMode")
 	notVerbose := fset.Bool("nv", false, "not log console")
 	newconfig := fset.Bool("newconfig", false, "not load existing config.json")
-	logLevel := fset.Int("loglevel", 0, "0:info 1:warn 2:error 3:debug")
+	logLevel := fset.Int("loglevel", 1, "0:debug 1:info 2:warn 3:error")
-	if subCommand == "" { // no subcommand
+	maxLogSize := fset.Int("maxlogsize", 1024*1024, "default 1MB")
-		fset.Parse(os.Args[1:])
+	if cmd == "" {
+		if subCommand == "" { // no subcommand
+			fset.Parse(os.Args[1:])
+		} else {
+			fset.Parse(os.Args[2:])
+		}
 	} else {
-		fset.Parse(os.Args[2:])
+		args := strings.Split(cmd, " ")
+		fset.Parse(args)
 	}
 
 	config := AppConfig{Enabled: 1}
 	config.PeerNode = *peerNode
 	config.DstHost = *dstIP
+	config.Whitelist = *whiteList
 	config.DstPort = *dstPort
 	config.SrcPort = *srcPort
 	config.Protocol = *protocol
+	config.UnderlayProtocol = *underlayProtocol
+	config.PunchPriority = *punchPriority
 	config.AppName = *appName
+	config.RelayNode = *relayNode
+	if *installPath != "" {
+		defaultInstallPath = *installPath
+	}
+	if subCommand == "install" {
+		if err := os.MkdirAll(defaultInstallPath, 0775); err != nil {
+			gLog.e("parseParams MkdirAll %s error:%s", defaultInstallPath, err)
+			return
+		}
+		if err := os.Chdir(defaultInstallPath); err != nil {
+			gLog.e("parseParams Chdir error:%s", err)
+			return
+		}
+	}
 	if !*newconfig {
 		gConf.load() // load old config. otherwise will clear all apps
 	}
-	if config.SrcPort != 0 {
+	if config.SrcPort != 0 { // filter memapp
 		gConf.add(config, true)
 	}
 	// gConf.mtx.Lock() // when calling this func it's single-thread no lock
@@ -217,7 +473,7 @@ func parseParams(subCommand string) {
 			gConf.Network.ShareBandwidth = *shareBandwidth
 		}
 		if f.Name == "node" {
-			gConf.Network.Node = *node
+			gConf.setNode(*node)
 		}
 		if f.Name == "serverhost" {
 			gConf.Network.ServerHost = *serverHost
@@ -225,43 +481,62 @@ func parseParams(subCommand string) {
 		if f.Name == "loglevel" {
 			gConf.LogLevel = *logLevel
 		}
-		if f.Name == "tcpport" {
+		if f.Name == "maxlogsize" {
-			gConf.Network.TCPPort = *tcpPort
+			gConf.MaxLogSize = *maxLogSize
+		}
+		if f.Name == "publicipport" {
+			gConf.Network.PublicIPPort = *publicIPPort
 		}
 		if f.Name == "token" {
-			gConf.Network.Token = *token
+			gConf.setToken(*token)
+		}
+		if f.Name == "serverport" {
+			gConf.Network.ServerPort = *serverPort
+		}
+		if f.Name == "insecure" {
+			gConf.TLSInsecureSkipVerify = *insecure
 		}
 	})
-
+	// set default value
 	if gConf.Network.ServerHost == "" {
 		gConf.Network.ServerHost = *serverHost
 	}
+	if gConf.Network.ServerPort == 0 {
+		gConf.Network.ServerPort = *serverPort
+	}
 	if *node != "" {
-		if len(*node) < MinNodeNameLen {
+		gConf.setNode(*node)
-			gLog.Println(LvERROR, ErrNodeTooShort)
-			os.Exit(9)
-		}
-		gConf.Network.Node = *node
 	} else {
+		envNode := os.Getenv("OPENP2P_NODE")
+		if envNode != "" {
+			gConf.setNode(envNode)
+		}
 		if gConf.Network.Node == "" { // if node name not set. use os.Hostname
-			gConf.Network.Node = defaultNodeName()
+			gConf.setNode(defaultNodeName())
 		}
 	}
-	if gConf.Network.TCPPort == 0 {
+	if gConf.Network.PublicIPPort == 0 {
-		if *tcpPort == 0 {
+		if *publicIPPort == 0 {
-			p := int(nodeNameToID(gConf.Network.Node)%15000 + 50000)
+			p := int(gConf.nodeID()%8192 + 1025)
-			tcpPort = &p
+			publicIPPort = &p
+		}
+		gConf.Network.PublicIPPort = *publicIPPort
+	}
+	if *token == 0 {
+		envToken := os.Getenv("OPENP2P_TOKEN")
+		if envToken != "" {
+			if n, err := strconv.ParseUint(envToken, 10, 64); n != 0 && err == nil {
+				gConf.setToken(n)
+			}
 		}
-		gConf.Network.TCPPort = *tcpPort
 	}
 
-	gConf.Network.ServerPort = *serverPort
+	gConf.Network.natDetectPort1 = NATDetectPort1
-	gConf.Network.UDPPort1 = UDPPort1
+	gConf.Network.natDetectPort2 = NATDetectPort2
-	gConf.Network.UDPPort2 = UDPPort2
 	gLog.setLevel(LogLevel(gConf.LogLevel))
+	gLog.setMaxSize(int64(gConf.MaxLogSize))
 	if *notVerbose {
 		gLog.setMode(LogFile)
 	}
-	// gConf.mtx.Unlock()
 	gConf.save()
 }

core/config_test.go

@@ -0,0 +1,178 @@
+package openp2p
+
+import (
+	"encoding/json"
+	"testing"
+)
+
+func TestSetSDWAN_ChangeNode(t *testing.T) {
+	conf := Config{}
+	sdwanInfo := SDWANInfo{}
+	sdwanStr := `{"id":1312667996276071700,"name":"network1","gateway":"10.2.3.254/24","mode":"fullmesh","centralNode":"n1-stable","enable":1,"Nodes":[{"name":"222-debug","ip":"10.2.3.13"},{"name":"222stable","ip":"10.2.3.222"},{"name":"5800-debug","ip":"10.2.3.56"},{"name":"Mate60pro","ip":"10.2.3.60"},{"name":"Mymatepad2023","ip":"10.2.3.23"},{"name":"n1-stable","ip":"10.2.3.29","resource":"192.168.3.0/24"},{"name":"tony-stable","ip":"10.2.3.4","resource":"10.1.0.0/16"}]}`
+	if err := json.Unmarshal([]byte(sdwanStr), &sdwanInfo); err != nil {
+		t.Errorf("unmarshal error")
+		return
+	}
+
+	conf.setSDWAN(sdwanInfo)
+	if len(conf.getDelNodes()) > 0 {
+		t.Errorf("getDelNodes error")
+		return
+	}
+	if len(conf.getAddNodes()) != 7 {
+		t.Errorf("getAddNodes error")
+		return
+	}
+	sdwanInfo2 := SDWANInfo{}
+	sdwanStr = `{"id":1312667996276071700,"name":"network1","gateway":"10.2.3.254/24","mode":"fullmesh","centralNode":"n1-stable","enable":1,"Nodes":[{"name":"222-debug","ip":"10.2.3.13"},{"name":"222stable","ip":"10.2.3.222"},{"name":"5800-debug","ip":"10.2.3.56"},{"name":"Mate60pro","ip":"10.2.3.60"},{"name":"Mymatepad2023","ip":"10.2.3.23"},{"name":"n1-stable","ip":"10.2.3.29","resource":"192.168.3.0/24"}]}`
+	if err := json.Unmarshal([]byte(sdwanStr), &sdwanInfo2); err != nil {
+		t.Errorf("unmarshal error")
+		return
+	}
+
+	conf.setSDWAN(sdwanInfo2)
+	diff := conf.getDelNodes()
+	if len(diff) != 1 && diff[0].IP != "10.2.3.4" {
+		t.Errorf("getDelNodes error")
+		return
+	}
+	sdwanInfo3 := SDWANInfo{}
+	sdwanStr = `{"id":1312667996276071700,"name":"network1","gateway":"10.2.3.254/24","mode":"fullmesh","centralNode":"n1-stable","enable":1,"Nodes":[{"name":"222-debug","ip":"10.2.3.13"},{"name":"222stable","ip":"10.2.3.222"},{"name":"5800-debug","ip":"10.2.3.56"},{"name":"Mymatepad2023","ip":"10.2.3.23"},{"name":"n1-stable","ip":"10.2.3.29","resource":"192.168.3.0/24"}]}`
+	if err := json.Unmarshal([]byte(sdwanStr), &sdwanInfo3); err != nil {
+		t.Errorf("unmarshal error")
+		return
+	}
+
+	conf.setSDWAN(sdwanInfo3)
+	diff = conf.getDelNodes()
+	if len(diff) != 1 && diff[0].IP != "10.2.3.60" {
+		t.Errorf("getDelNodes error")
+		return
+	}
+	// add new node
+	sdwanInfo4 := SDWANInfo{}
+	sdwanStr = `{"id":1312667996276071700,"name":"network1","gateway":"10.2.3.254/24","mode":"fullmesh","centralNode":"n1-stable","enable":1,"Nodes":[{"name":"222-debug","ip":"10.2.3.13"},{"name":"222stable","ip":"10.2.3.222"},{"name":"5800-debug","ip":"10.2.3.56"},{"name":"Mate60pro","ip":"10.2.3.60"},{"name":"Mymatepad2023","ip":"10.2.3.23"},{"name":"n1-stable","ip":"10.2.3.29","resource":"192.168.3.0/24"}]}`
+	if err := json.Unmarshal([]byte(sdwanStr), &sdwanInfo4); err != nil {
+		t.Errorf("unmarshal error")
+		return
+	}
+
+	conf.setSDWAN(sdwanInfo4)
+	diff = conf.getDelNodes()
+	if len(diff) > 0 {
+		t.Errorf("getDelNodes error")
+		return
+	}
+	diff = conf.getAddNodes()
+	if len(diff) != 1 && diff[0].IP != "10.2.3.60" {
+		t.Errorf("getAddNodes error")
+		return
+	}
+}
+
+func TestSetSDWAN_ChangeNodeIP(t *testing.T) {
+	conf := Config{}
+	sdwanInfo := SDWANInfo{}
+	sdwanStr := `{"id":1312667996276071700,"name":"network1","gateway":"10.2.3.254/24","mode":"fullmesh","centralNode":"n1-stable","enable":1,"Nodes":[{"name":"222-debug","ip":"10.2.3.13"},{"name":"222stable","ip":"10.2.3.222"},{"name":"5800-debug","ip":"10.2.3.56"},{"name":"Mate60pro","ip":"10.2.3.60"},{"name":"Mymatepad2023","ip":"10.2.3.23"},{"name":"n1-stable","ip":"10.2.3.29","resource":"192.168.3.0/24"},{"name":"tony-stable","ip":"10.2.3.4","resource":"10.1.0.0/16"}]}`
+	if err := json.Unmarshal([]byte(sdwanStr), &sdwanInfo); err != nil {
+		t.Errorf("unmarshal error")
+		return
+	}
+
+	conf.setSDWAN(sdwanInfo)
+	if len(conf.getDelNodes()) > 0 {
+		t.Errorf("getDelNodes error")
+		return
+	}
+	sdwanInfo2 := SDWANInfo{}
+	sdwanStr = `{"id":1312667996276071700,"name":"network1","gateway":"10.2.3.254/24","mode":"fullmesh","centralNode":"n1-stable","enable":1,"Nodes":[{"name":"222-debug","ip":"10.2.3.13"},{"name":"222stable","ip":"10.2.3.222"},{"name":"5800-debug","ip":"10.2.3.56"},{"name":"Mate60pro","ip":"10.2.3.60"},{"name":"Mymatepad2023","ip":"10.2.3.23"},{"name":"n1-stable","ip":"10.2.3.29","resource":"192.168.3.0/24"},{"name":"tony-stable","ip":"10.2.3.44","resource":"10.1.0.0/16"}]}`
+	if err := json.Unmarshal([]byte(sdwanStr), &sdwanInfo2); err != nil {
+		t.Errorf("unmarshal error")
+		return
+	}
+
+	conf.setSDWAN(sdwanInfo2)
+	diff := conf.getDelNodes()
+	if len(diff) != 1 && diff[0].IP != "10.2.3.4" {
+		t.Errorf("getDelNodes error")
+		return
+	}
+	diff = conf.getAddNodes()
+	if len(diff) != 1 || diff[0].IP != "10.2.3.44" {
+		t.Errorf("getAddNodes error")
+		return
+	}
+}
+func TestSetSDWAN_ClearAll(t *testing.T) {
+	conf := Config{}
+	sdwanInfo := SDWANInfo{}
+	sdwanStr := `{"id":1312667996276071700,"name":"network1","gateway":"10.2.3.254/24","mode":"fullmesh","centralNode":"n1-stable","enable":1,"Nodes":[{"name":"222-debug","ip":"10.2.3.13"},{"name":"222stable","ip":"10.2.3.222"},{"name":"5800-debug","ip":"10.2.3.56"},{"name":"Mate60pro","ip":"10.2.3.60"},{"name":"Mymatepad2023","ip":"10.2.3.23"},{"name":"n1-stable","ip":"10.2.3.29","resource":"192.168.3.0/24"},{"name":"tony-stable","ip":"10.2.3.4","resource":"10.1.0.0/16"}]}`
+	if err := json.Unmarshal([]byte(sdwanStr), &sdwanInfo); err != nil {
+		t.Errorf("unmarshal error")
+		return
+	}
+
+	conf.setSDWAN(sdwanInfo)
+	if len(conf.getDelNodes()) > 0 {
+		t.Errorf("getDelNodes error")
+		return
+	}
+	sdwanInfo2 := SDWANInfo{}
+	sdwanStr = `{"Nodes":null}`
+	if err := json.Unmarshal([]byte(sdwanStr), &sdwanInfo2); err != nil {
+		t.Errorf("unmarshal error")
+		return
+	}
+
+	conf.setSDWAN(sdwanInfo2)
+	diff := conf.getDelNodes()
+	if len(diff) != 7 {
+		t.Errorf("getDelNodes error")
+		return
+	}
+	diff = conf.getAddNodes()
+	if len(diff) != 0 {
+		t.Errorf("getAddNodes error")
+		return
+	}
+}
+func TestSetSDWAN_ChangeNodeResource(t *testing.T) {
+	conf := Config{}
+	sdwanInfo := SDWANInfo{}
+	sdwanStr := `{"id":1312667996276071700,"name":"network1","gateway":"10.2.3.254/24","mode":"fullmesh","centralNode":"n1-stable","enable":1,"Nodes":[{"name":"222-debug","ip":"10.2.3.13"},{"name":"222stable","ip":"10.2.3.222"},{"name":"5800-debug","ip":"10.2.3.56"},{"name":"Mate60pro","ip":"10.2.3.60"},{"name":"Mymatepad2023","ip":"10.2.3.23"},{"name":"n1-stable","ip":"10.2.3.29","resource":"192.168.3.0/24"},{"name":"tony-stable","ip":"10.2.3.4","resource":"10.1.0.0/16"}]}`
+	if err := json.Unmarshal([]byte(sdwanStr), &sdwanInfo); err != nil {
+		t.Errorf("unmarshal error")
+		return
+	}
+
+	conf.setSDWAN(sdwanInfo)
+	if len(conf.getDelNodes()) > 0 {
+		t.Errorf("getDelNodes error")
+		return
+	}
+	sdwanInfo2 := SDWANInfo{}
+	sdwanStr = `{"id":1312667996276071700,"name":"network1","gateway":"10.2.3.254/24","mode":"fullmesh","centralNode":"n1-stable","enable":1,"Nodes":[{"name":"222-debug","ip":"10.2.3.13"},{"name":"222stable","ip":"10.2.3.222"},{"name":"5800-debug","ip":"10.2.3.56"},{"name":"Mate60pro","ip":"10.2.3.60"},{"name":"Mymatepad2023","ip":"10.2.3.23"},{"name":"n1-stable","ip":"10.2.3.29","resource":"192.168.3.0/24"},{"name":"tony-stable","ip":"10.2.3.4","resource":"10.11.0.0/16"}]}`
+	if err := json.Unmarshal([]byte(sdwanStr), &sdwanInfo2); err != nil {
+		t.Errorf("unmarshal error")
+		return
+	}
+
+	conf.setSDWAN(sdwanInfo2)
+	diff := conf.getDelNodes()
+	if len(diff) != 1 && diff[0].IP != "10.2.3.4" {
+		t.Errorf("getDelNodes error")
+		return
+	}
+	diff = conf.getAddNodes()
+	if len(diff) != 1 || diff[0].Resource != "10.11.0.0/16" {
+		t.Errorf("getAddNodes error")
+		return
+	}
+}
+
+func TestInetAtoN(t *testing.T) {
+	ipa, _ := inetAtoN("121.5.147.4")
+	t.Log(ipa)
+	ipa, _ = inetAtoN("121.5.147.4/32")
+	t.Log(ipa)
+}

core/daemon.go

@@ -1,115 +1,186 @@
-package openp2p
+package openp2p
-
+
-import (
+import (
-	"fmt"
+	"os"
-	"os"
+	"path/filepath"
-	"path/filepath"
+	"runtime"
-	"time"
+	"time"
-
+
-	"github.com/kardianos/service"
+	"github.com/openp2p-cn/service"
-)
+)
-
+
-type daemon struct {
+type daemon struct {
-	running bool
+	running bool
-	proc    *os.Process
+	proc    *os.Process
-}
+}
-
+
-func (d *daemon) Start(s service.Service) error {
+func (d *daemon) Start(s service.Service) error {
-	gLog.Println(LvINFO, "daemon start")
+	gLog.i("system service start")
-	return nil
+	return nil
-}
+}
-
+
-func (d *daemon) Stop(s service.Service) error {
+func (d *daemon) Stop(s service.Service) error {
-	gLog.Println(LvINFO, "service stop")
+	gLog.i("system service stop")
-	d.running = false
+	d.running = false
-	if d.proc != nil {
+	if d.proc != nil {
-		gLog.Println(LvINFO, "stop worker")
+		gLog.i("stop worker")
-		d.proc.Kill()
+		d.proc.Kill()
-	}
+	}
-	if service.Interactive() {
+	if service.Interactive() {
-		gLog.Println(LvINFO, "stop daemon")
+		gLog.i("stop daemon")
-		os.Exit(0)
+		os.Exit(0)
-	}
+	}
-	return nil
+	return nil
-}
+}
-
+
-func (d *daemon) run() {
+func (d *daemon) run() {
-	gLog.Println(LvINFO, "daemon run start")
+	gLog.close()
-	defer gLog.Println(LvINFO, "daemon run end")
+	baseDir := filepath.Dir(os.Args[0])
-	d.running = true
+	gLog = NewLogger(baseDir, "daemon", LogLevel(gConf.LogLevel), 1024*1024, LogFile|LogConsole)
-	binPath, _ := os.Executable()
+	gLog.i("daemon run start")
-	mydir, err := os.Getwd()
+	defer gLog.i("daemon run end")
-	if err != nil {
+	d.running = true
-		fmt.Println(err)
+	binPath, _ := os.Executable()
-	}
+	conf := &service.Config{
-	gLog.Println(LvINFO, mydir)
+		Name:        ProductName,
-	conf := &service.Config{
+		DisplayName: ProductName,
-		Name:        ProducnName,
+		Description: ProductName,
-		DisplayName: ProducnName,
+		Executable:  binPath,
-		Description: ProducnName,
+	}
-		Executable:  binPath,
+
-	}
+	s, _ := service.New(d, conf)
-
+	go s.Run()
-	s, _ := service.New(d, conf)
+	var args []string
-	go s.Run()
+	// rm -d parameter
-	var args []string
+	for i := 0; i < len(os.Args); i++ {
-	// rm -d parameter
+		if os.Args[i] == "-d" {
-	for i := 0; i < len(os.Args); i++ {
+			args = append(os.Args[0:i], os.Args[i+1:]...)
-		if os.Args[i] == "-d" {
+			break
-			args = append(os.Args[0:i], os.Args[i+1:]...)
+		}
-			break
+	}
-		}
+
-	}
+	args = append(args, "-nv")
-	args = append(args, "-nv")
+	for {
-	for {
+		// start worker
-		// start worker
+		tmpDump := filepath.Join(filepath.Dir(binPath), "log", "dump.log.tmp")
-		tmpDump := filepath.Join("log", "dump.log.tmp")
+		dumpFile := filepath.Join(filepath.Dir(binPath), "log", "dump.log")
-		dumpFile := filepath.Join("log", "dump.log")
+		// f, err := os.Create(filepath.Join(tmpDump))
-		f, err := os.Create(filepath.Join(tmpDump))
+		f, err := os.OpenFile(filepath.Join(tmpDump), os.O_CREATE|os.O_WRONLY|os.O_TRUNC, 0775)
-		if err != nil {
+		if err != nil {
-			gLog.Printf(LvERROR, "start worker error:%s", err)
+			gLog.e("OpenFile %s error:%s", tmpDump, err)
-			return
+			return
-		}
+		}
-		gLog.Println(LvINFO, "start worker process, args:", args)
+		gLog.i("start worker process, args:%v", args)
-		execSpec := &os.ProcAttr{Env: append(os.Environ(), "GOTRACEBACK=crash"), Files: []*os.File{os.Stdin, os.Stdout, f}}
+		execSpec := &os.ProcAttr{Env: append(os.Environ(), "GOTRACEBACK=crash"), Files: []*os.File{os.Stdin, os.Stdout, f}}
-		p, err := os.StartProcess(binPath, args, execSpec)
+		lastRebootTime := time.Now()
-		if err != nil {
+		p, err := os.StartProcess(binPath, args, execSpec)
-			gLog.Printf(LvERROR, "start worker error:%s", err)
+		if err != nil {
-			return
+			gLog.e("start worker error:%s", err)
-		}
+			return
-		d.proc = p
+		}
-		_, _ = p.Wait()
+		d.proc = p
-		f.Close()
+		processState, err := p.Wait()
-		time.Sleep(time.Second)
+		if err != nil {
-		err = os.Rename(tmpDump, dumpFile)
+			gLog.e("wait process error:%s", err)
-		if err != nil {
+		}
-			gLog.Printf(LvERROR, "rename dump error:%s", err)
+
-		}
+		if processState != nil {
-		if !d.running {
+			exitCode := processState.ExitCode()
-			return
+			gLog.i("worker process exited with code: %d", exitCode)
-		}
+
-		gLog.Printf(LvERROR, "worker stop, restart it after 10s")
+			if exitCode == 9 {
-		time.Sleep(time.Second * 10)
+				gLog.i("worker process update with code: %d", exitCode)
-	}
+				// os.Exit(9) // old client installed system service will not auto restart. fuck
-}
+			}
-
+		}
-func (d *daemon) Control(ctrlComm string, exeAbsPath string, args []string) error {
+		// Write the current time to the end of the dump file
-	svcConfig := &service.Config{
+		currentTime := time.Now().Format("2006-01-02 15:04:05")
-		Name:        ProducnName,
+		_, err = f.WriteString("\nProcess ended at: " + currentTime + "\n")
-		DisplayName: ProducnName,
+		if err != nil {
-		Description: ProducnName,
+			gLog.e("Failed to write time to dump file: %s", err)
-		Executable:  exeAbsPath,
+		}
-		Arguments:   args,
+
-	}
+		f.Close()
-
+		time.Sleep(time.Second)
-	s, e := service.New(d, svcConfig)
+		err = os.Rename(tmpDump, dumpFile)
-	if e != nil {
+		if err != nil {
-		return e
+			gLog.e("rename dump error:%s", err)
-	}
+		}
-	e = service.Control(s, ctrlComm)
+		if !d.running {
-	if e != nil {
+			return
-		return e
+		}
-	}
+		if time.Since(lastRebootTime) < time.Second*10 {
-
+			gLog.e("worker stop, restart it after 10s")
-	return nil
+			time.Sleep(time.Second * 10)
-}
+		}
+
+	}
+}
+
+func (d *daemon) Control(ctrlComm string, exeAbsPath string, args []string) error {
+	svcConfig := getServiceConfig(exeAbsPath, args)
+
+	s, e := service.New(d, svcConfig)
+	if e != nil {
+		return e
+	}
+	e = service.Control(s, ctrlComm)
+	if e != nil {
+		return e
+	}
+
+	return nil
+}
+
+func getServiceConfig(exeAbsPath string, args []string) *service.Config {
+	config := &service.Config{
+		Name:        ProductName,
+		DisplayName: ProductName,
+		Description: ProductName,
+		Executable:  exeAbsPath,
+		Arguments:   args,
+		Option:      make(map[string]interface{}),
+	}
+
+	if runtime.GOOS == "windows" {
+		setupWindowsConfig(config)
+	} else {
+		setupLinuxConfig(config)
+	}
+
+	return config
+}
+
+func setupWindowsConfig(config *service.Config) {
+	failureActions := []map[string]interface{}{
+		{
+			"Type":  "restart",
+			"Delay": "10000",
+		},
+		{
+			"Type":  "restart",
+			"Delay": "10000",
+		},
+		{
+			"Type":  "restart",
+			"Delay": "10000",
+		},
+	}
+
+	config.Option = map[string]interface{}{
+		"OnFailure":            "restart",
+		"OnFailureDelay":       "10s",
+		"OnFailureResetPeriod": "3600",
+		"FailureActions":       failureActions,
+		"DelayedAutoStart":     true,
+	}
+}
+
+func setupLinuxConfig(config *service.Config) {
+	config.Option = map[string]interface{}{
+		"Restart":           "always",
+		"RestartSec":        "10",
+		"StartLimitBurst":   64,
+		"SuccessExitStatus": "1 2 8 SIGKILL",
+	}
+}

core/errorcode.go

@@ -8,12 +8,29 @@ import (
 var (
 	// ErrorS2S string = "s2s is not supported"
 	// ErrorHandshake string = "handshake error"
-	ErrorS2S                = errors.New("s2s is not supported")
+	ErrorS2S                 = errors.New("s2s is not supported")
-	ErrorHandshake          = errors.New("handshake error")
+	ErrorHandshake           = errors.New("handshake error")
-	ErrorNewUser            = errors.New("new user")
+	ErrorNewUser             = errors.New("new user")
-	ErrorLogin              = errors.New("user or password not correct")
+	ErrorLogin               = errors.New("user or password not correct")
-	ErrNodeTooShort         = errors.New("node name too short, it must >=8 charaters")
+	ErrNodeTooShort          = errors.New("node name too short, it must >=8 charaters")
-	ErrPeerOffline          = errors.New("peer offline")
+	ErrReadDB                = errors.New("read db error")
-	ErrMsgFormat            = errors.New("message format wrong")
+	ErrNoUpdate              = errors.New("there are currently no updates available")
-	ErrVersionNotCompatible = errors.New("version not compatible")
+	ErrPeerOffline           = errors.New("peer offline")
+	ErrNetwork               = errors.New("network error")
+	ErrMsgFormat             = errors.New("message format wrong")
+	ErrVersionNotCompatible  = errors.New("version not compatible")
+	ErrOverlayConnDisconnect = errors.New("overlay connection is disconnected")
+	ErrConnectRelayNode      = errors.New("connect relay node error")
+	ErrConnectPublicV4       = errors.New("connect public ipv4 error")
+	ErrMsgChannelNotFound    = errors.New("message channel not found")
+	ErrRelayTunnelNotFound   = errors.New("relay tunnel not found")
+	ErrSymmetricLimit        = errors.New("symmetric limit")
+	ErrForceRelay            = errors.New("force relay")
+	ErrPeerConnectRelay      = errors.New("peer connect relayNode error")
+	ErrBuildTunnelBusy       = errors.New("build tunnel busy")
+	ErrMemAppTunnelNotFound  = errors.New("memapp tunnel not found")
+	ErrRemoteServiceUnable   = errors.New("remote service unable")
+	ErrAppWithoutTunnel      = errors.New("p2papp has no available tunnel")
+	ErrWriteWindowFull       = errors.New("writeWindow full")
+	ErrHeaderDataLen         = errors.New("header datalen error")
 )

core/handlepush.go

@@ -1,282 +1,578 @@
-package openp2p
+package openp2p
-
+
-import (
+import (
-	"bytes"
+	"bytes"
-	"encoding/binary"
+	"encoding/binary"
-	"encoding/json"
+	"encoding/json"
-	"fmt"
+	"errors"
-	"os"
+	"fmt"
-	"os/exec"
+	"net"
-	"path/filepath"
+	"os"
-	"time"
+	"path/filepath"
-)
+	"reflect"
-
+	"runtime"
-func handlePush(pn *P2PNetwork, subType uint16, msg []byte) error {
+	"runtime/pprof"
-	pushHead := PushHeader{}
+	"time"
-	err := binary.Read(bytes.NewReader(msg[openP2PHeaderSize:openP2PHeaderSize+PushHeaderSize]), binary.LittleEndian, &pushHead)
+
-	if err != nil {
+	"github.com/openp2p-cn/totp"
-		return err
+)
-	}
+
-	gLog.Printf(LvDEBUG, "handle push msg type:%d, push header:%+v", subType, pushHead)
+func handlePush(subType uint16, msg []byte) error {
-	switch subType {
+	pushHead := PushHeader{}
-	case MsgPushConnectReq: // TODO: handle a msg move to a new function
+	err := binary.Read(bytes.NewReader(msg[openP2PHeaderSize:openP2PHeaderSize+PushHeaderSize]), binary.LittleEndian, &pushHead)
-		req := PushConnectReq{}
+	if err != nil {
-		err := json.Unmarshal(msg[openP2PHeaderSize+PushHeaderSize:], &req)
+		return err
-		if err != nil {
+	}
-			gLog.Printf(LvERROR, "wrong MsgPushConnectReq:%s", err)
+	// gLog.d("handle push msg type:%d, push header:%+v", subType, pushHead)
-			return err
+	switch subType {
-		}
+	case MsgPushConnectReq:
-		gLog.Printf(LvINFO, "%s is connecting...", req.From)
+		err = handleConnectReq(msg)
-		gLog.Println(LvDEBUG, "push connect response to ", req.From)
+	case MsgPushRsp:
-		if compareVersion(req.Version, LeastSupportVersion) == LESS {
+		rsp := PushRsp{}
-			gLog.Println(LvERROR, ErrVersionNotCompatible.Error(), ":", req.From)
+		if err = json.Unmarshal(msg[openP2PHeaderSize:], &rsp); err != nil {
-			rsp := PushConnectRsp{
+			gLog.e("Unmarshal pushRsp:%s", err)
-				Error:  10,
+			return err
-				Detail: ErrVersionNotCompatible.Error(),
+		}
-				To:     req.From,
+		if rsp.Error == 0 {
-				From:   pn.config.Node,
+			gLog.dev("push ok, detail:%s", rsp.Detail)
-			}
+		} else {
-			pn.push(req.From, MsgPushConnectRsp, rsp)
+			gLog.e("push error:%d, detail:%s", rsp.Error, rsp.Detail)
-			return ErrVersionNotCompatible
+		}
-		}
+	case MsgPushAddRelayTunnelReq:
-		// verify totp token or token
+		req := AddRelayTunnelReq{}
-		if VerifyTOTP(req.Token, pn.config.Token, time.Now().Unix()+(pn.serverTs-pn.localTs)) || // localTs may behind, auto adjust ts
+		if err = json.Unmarshal(msg[openP2PHeaderSize+PushHeaderSize:], &req); err != nil {
-			VerifyTOTP(req.Token, pn.config.Token, time.Now().Unix()) {
+			gLog.e("Unmarshal %v:%s", reflect.TypeOf(req), err)
-			gLog.Printf(LvINFO, "Access Granted\n")
+			return err
-			config := AppConfig{}
+		}
-			config.peerNatType = req.NatType
+		config := AppConfig{}
-			config.peerConeNatPort = req.ConeNatPort
+		config.PeerNode = req.RelayName
-			config.peerIP = req.FromIP
+		config.peerToken = req.RelayToken
-			config.PeerNode = req.From
+		config.relayMode = req.RelayMode
-			config.peerVersion = req.Version
+		config.PunchPriority = req.PunchPriority
-			config.fromToken = req.Token
+		config.UnderlayProtocol = req.UnderlayProtocol
-			config.peerIPv6 = req.IPv6
+		go func(r AddRelayTunnelReq) {
-			config.hasIPv4 = req.HasIPv4
+			t, errDt := GNetwork.addDirectTunnel(config, 0, nil)
-			config.hasUPNPorNATPMP = req.HasUPNPorNATPMP
+			if errDt == nil && t != nil {
-			config.linkMode = req.LinkMode
+				// notify peer relay ready
-			config.isUnderlayServer = req.IsUnderlayServer
+				msg := TunnelMsg{ID: t.id}
-			// share relay node will limit bandwidth
+				GNetwork.push(r.From, MsgPushAddRelayTunnelRsp, msg)
-			if req.Token != pn.config.Token {
+				appConfig := config
-				gLog.Printf(LvINFO, "set share bandwidth %d mbps", pn.config.ShareBandwidth)
+				appConfig.PeerNode = req.From
-				config.shareBandwidth = pn.config.ShareBandwidth
+			} else {
-			}
+				gLog.w("addDirectTunnel error:%s", errDt)
-			// go pn.AddTunnel(config, req.ID)
+				GNetwork.push(r.From, MsgPushAddRelayTunnelRsp, "error") // compatible with old version client, trigger unmarshal error
-			go pn.addDirectTunnel(config, req.ID)
+			}
-			break
+		}(req)
-		}
+	case MsgPushServerSideSaveMemApp:
-		gLog.Println(LvERROR, "Access Denied:", req.From)
+		req := ServerSideSaveMemApp{}
-		rsp := PushConnectRsp{
+		if err = json.Unmarshal(msg[openP2PHeaderSize+PushHeaderSize:], &req); err != nil {
-			Error:  1,
+			gLog.e("Unmarshal %v:%s", reflect.TypeOf(req), err)
-			Detail: fmt.Sprintf("connect to %s error: Access Denied", pn.config.Node),
+			return err
-			To:     req.From,
+		}
-			From:   pn.config.Node,
+		gLog.d("handle MsgPushServerSideSaveMemApp:%s", prettyJson(req))
-		}
+		if req.RelayIndex > uint32(gConf.sdwan.TunnelNum-1) {
-		pn.push(req.From, MsgPushConnectRsp, rsp)
+			return errors.New("wrong relay index")
-	case MsgPushRsp:
+		}
-		rsp := PushRsp{}
+		var existTunnel *P2PTunnel
-		err := json.Unmarshal(msg[openP2PHeaderSize:], &rsp)
+		i, ok := GNetwork.allTunnels.Load(req.TunnelID)
-		if err != nil {
+		if !ok {
-			gLog.Printf(LvERROR, "wrong pushRsp:%s", err)
+			time.Sleep(time.Millisecond * 3000)
-			return err
+			i, ok = GNetwork.allTunnels.Load(req.TunnelID) // retry sometimes will receive MsgPushServerSideSaveMemApp but p2ptunnel not store yet.
-		}
+			if !ok {
-		if rsp.Error == 0 {
+				gLog.e("handle MsgPushServerSideSaveMemApp error:%s", ErrMemAppTunnelNotFound)
-			gLog.Printf(LvDEBUG, "push ok, detail:%s", rsp.Detail)
+				return ErrMemAppTunnelNotFound
-		} else {
+			}
-			gLog.Printf(LvERROR, "push error:%d, detail:%s", rsp.Error, rsp.Detail)
+		}
-		}
+		existTunnel = i.(*P2PTunnel)
-	case MsgPushAddRelayTunnelReq:
+		peerID := NodeNameToID(req.From)
-		req := AddRelayTunnelReq{}
+		appIdx := peerID
-		err := json.Unmarshal(msg[openP2PHeaderSize+PushHeaderSize:], &req)
+		if req.SrcPort != 0 {
-		if err != nil {
+			appIdx = req.AppID
-			gLog.Printf(LvERROR, "wrong RelayNodeRsp:%s", err)
+		}
-			return err
+		existApp, appok := GNetwork.apps.Load(appIdx)
-		}
+		var app *p2pApp
-		config := AppConfig{}
+		if appok {
-		config.PeerNode = req.RelayName
+			app = existApp.(*p2pApp)
-		config.peerToken = req.RelayToken
+			if app.tunnelNum != int(req.TunnelNum) {
-		go func(r AddRelayTunnelReq) {
+				gLog.d("memapp tunnelNum changed from %d to %d", app.tunnelNum, req.TunnelNum)
-			t, errDt := pn.addDirectTunnel(config, 0)
+				GNetwork.DeleteApp(app.config)
-			if errDt == nil {
+				app = nil
-				// notify peer relay ready
+			}
-				msg := TunnelMsg{ID: t.id}
+		}
-				pn.push(r.From, MsgPushAddRelayTunnelRsp, msg)
+		if app != nil {
-			}
+			app.config.AppName = fmt.Sprintf("%d", peerID)
-
+			app.id = req.AppID
-		}(req)
+			app.key = req.AppKey
-	case MsgPushAPPKey:
+			app.PreCalcKeyBytes()
-		req := APPKeySync{}
+			app.relayMode[req.RelayIndex] = req.RelayMode
-		err := json.Unmarshal(msg[openP2PHeaderSize+PushHeaderSize:], &req)
+			app.hbTime[req.RelayIndex] = time.Now()
-		if err != nil {
+			app.SetTunnel(existTunnel, int(req.RelayIndex))
-			gLog.Printf(LvERROR, "wrong APPKeySync:%s", err)
+			if req.RelayTunnelID != 0 {
-			return err
+				app.SetRelayTunnelID(req.RelayTunnelID, int(req.RelayIndex)) // direct tunnel rtid=0, no need set rtid
-		}
+			}
-		SaveKey(req.AppID, req.AppKey)
+			gLog.d("found existing memapp, update it")
-	case MsgPushUpdate:
+		} else {
-		gLog.Println(LvINFO, "MsgPushUpdate")
+			appConfig := existTunnel.config
-		update(pn.config.ServerHost, pn.config.ServerPort) // download new version first, then exec ./openp2p update
+			appConfig.SrcPort = int(req.SrcPort)
-		targetPath := filepath.Join(defaultInstallPath, defaultBinName)
+			appConfig.Protocol = ""
-		args := []string{"update"}
+			appConfig.AppName = fmt.Sprintf("%d", peerID)
-		env := os.Environ()
+			appConfig.PeerNode = req.From
-		cmd := exec.Command(targetPath, args...)
+			app = &p2pApp{
-		cmd.Stdout = os.Stdout
+				id:      req.AppID,
-		cmd.Stderr = os.Stderr
+				config:  appConfig,
-		cmd.Stdin = os.Stdin
+				running: true,
-		cmd.Env = env
+				// asyncWriteChan: make(chan []byte, WriteDataChanSize),
-		err := cmd.Run()
+				key: req.AppKey,
-		if err == nil {
+			}
-			os.Exit(0)
+			app.PreCalcKeyBytes()
-		}
+			tunnelNum := 2
-		return err
+			if req.TunnelNum > uint32(tunnelNum) {
-	case MsgPushRestart:
+				tunnelNum = int(req.TunnelNum)
-		gLog.Println(LvINFO, "MsgPushRestart")
+			}
-		os.Exit(0)
+			app.Init(tunnelNum)
-		return err
+			app.relayMode[req.RelayIndex] = req.RelayMode
-	case MsgPushReportApps:
+			app.hbTime[req.RelayIndex] = time.Now()
-		gLog.Println(LvINFO, "MsgPushReportApps")
+			app.SetTunnel(existTunnel, int(req.RelayIndex))
-		req := ReportApps{}
+			if req.RelayTunnelID != 0 {
-		gConf.mtx.Lock()
+				app.SetRelayTunnelID(req.RelayTunnelID, int(req.RelayIndex))
-		defer gConf.mtx.Unlock()
+				app.relayNode[req.RelayIndex] = req.Node
-		for _, config := range gConf.Apps {
+			}
-			appActive := 0
+			app.Start(false)
-			relayNode := ""
+			GNetwork.apps.Store(appIdx, app)
-			relayMode := ""
+			gLog.d("store memapp %d %d", appIdx, req.SrcPort)
-			linkMode := LinkModeUDPPunch
+		}
-			i, ok := pn.apps.Load(fmt.Sprintf("%s%d", config.Protocol, config.SrcPort))
+
-			if ok {
+		return nil
-				app := i.(*p2pApp)
+	case MsgPushUpdate:
-				if app.isActive() {
+		gLog.i("MsgPushUpdate")
-					appActive = 1
+		err := update(gConf.Network.ServerHost, gConf.Network.ServerPort)
-				}
+		if err == nil {
-				relayNode = app.relayNode
+			if !isAndroid() {
-				relayMode = app.relayMode
+				os.Exit(9) // 9 tell daemon this exit because of update
-				linkMode = app.tunnel.linkModeWeb
+			}
-			}
+
-			appInfo := AppInfo{
+		}
-				AppName:     config.AppName,
+		return err
-				Error:       config.errMsg,
+	case MsgPushRestart:
-				Protocol:    config.Protocol,
+		gLog.i("MsgPushRestart")
-				SrcPort:     config.SrcPort,
+		if !isAndroid() {
-				RelayNode:   relayNode,
+			os.Exit(0)
-				RelayMode:   relayMode,
+		}
-				LinkMode:    linkMode,
+		return err
-				PeerNode:    config.PeerNode,
+	case MsgPushReportApps:
-				DstHost:     config.DstHost,
+		err = handleReportApps()
-				DstPort:     config.DstPort,
+	case MsgPushReportMemApps:
-				PeerUser:    config.PeerUser,
+		err = handleReportMemApps()
-				PeerIP:      config.peerIP,
+	case MsgPushReportLog:
-				PeerNatType: config.peerNatType,
+		err = handleLog(msg)
-				RetryTime:   config.retryTime.Local().Format("2006-01-02T15:04:05-0700"),
+	case MsgPushReportGoroutine:
-				ConnectTime: config.connectTime.Local().Format("2006-01-02T15:04:05-0700"),
+		err = handleReportGoroutine()
-				IsActive:    appActive,
+	case MsgPushReportHeap:
-				Enabled:     config.Enabled,
+		err = handleReportHeap()
-			}
+	case MsgPushCheckRemoteService:
-			req.Apps = append(req.Apps, appInfo)
+		err = handleCheckRemoteService(msg)
-		}
+	case MsgPushEditApp:
-		pn.write(MsgReport, MsgReportApps, &req)
+		err = handleEditApp(msg)
-	case MsgPushReportLog:
+	case MsgPushEditNode:
-		gLog.Println(LvINFO, "MsgPushReportLog")
+		gLog.i("MsgPushEditNode")
-		req := ReportLogReq{}
+		req := EditNode{}
-		err := json.Unmarshal(msg[openP2PHeaderSize:], &req)
+		if err = json.Unmarshal(msg[openP2PHeaderSize:], &req); err != nil {
-		if err != nil {
+			gLog.e("Unmarshal %v:%s  %s", reflect.TypeOf(req), err, string(msg[openP2PHeaderSize:]))
-			gLog.Printf(LvERROR, "wrong MsgPushReportLog:%s  %s", err, string(msg[openP2PHeaderSize:]))
+			return err
-			return err
+		}
-		}
+		gConf.setNode(req.NewName)
-		if req.FileName == "" {
+		gConf.setShareBandwidth(req.Bandwidth)
-			req.FileName = "openp2p.log"
+		if req.PublicIPPort != 0 {
-		}
+			gConf.Network.PublicIPPort = req.PublicIPPort
-		f, err := os.Open(filepath.Join("log", req.FileName))
+		}
-		if err != nil {
+		gConf.Forcev6 = (req.Forcev6 != 0)
-			gLog.Println(LvERROR, "read log file error:", err)
+		gLog.i("set forcev6 to %v", gConf.Forcev6)
-			break
+		gConf.save()
-		}
+		os.Exit(0)
-		fi, err := f.Stat()
+	case MsgPushSwitchApp:
-		if err != nil {
+		gLog.i("MsgPushSwitchApp")
-			break
+		app := AppInfo{}
-		}
+		if err = json.Unmarshal(msg[openP2PHeaderSize:], &app); err != nil {
-		if req.Offset == 0 && fi.Size() > 4096 {
+			gLog.e("Unmarshal %v:%s  %s", reflect.TypeOf(app), err, string(msg[openP2PHeaderSize:]))
-			req.Offset = fi.Size() - 4096
+			return err
-		}
+		}
-		if req.Len <= 0 {
+		config := AppConfig{PeerNode: app.PeerNode, Enabled: app.Enabled, SrcPort: app.SrcPort, Protocol: app.Protocol}
-			req.Len = 4096
+		gLog.i("%s switch to %d", app.AppName, app.Enabled)
-		}
+		gConf.switchApp(config, app.Enabled)
-		f.Seek(req.Offset, 0)
+		if app.Enabled == 0 {
-		if req.Len > 1024*1024 { // too large
+			// disable APP
-			break
+			GNetwork.DeleteApp(config)
-		}
+		}
-		buff := make([]byte, req.Len)
+	case MsgPushDstNodeOnline:
-		readLength, err := f.Read(buff)
+		req := PushDstNodeOnline{}
-		f.Close()
+		if err = json.Unmarshal(msg[openP2PHeaderSize:], &req); err != nil {
-		if err != nil {
+			gLog.e("Unmarshal %v:%s  %s", reflect.TypeOf(req), err, string(msg[openP2PHeaderSize:]))
-			gLog.Println(LvERROR, "read log content error:", err)
+			return err
-			break
+		}
-		}
+		gLog.i("%s online, retryApp", req.Node)
-		rsp := ReportLogRsp{}
+		gConf.retryApp(req.Node)
-		rsp.Content = string(buff[:readLength])
+	case MsgPushSpecTunnel:
-		rsp.FileName = req.FileName
+		req := SpecTunnel{}
-		rsp.Total = fi.Size()
+		if err = json.Unmarshal(msg[openP2PHeaderSize:], &req); err != nil {
-		rsp.Len = req.Len
+			gLog.e("Unmarshal %v:%s  %s", reflect.TypeOf(req), err, string(msg[openP2PHeaderSize:]))
-		pn.write(MsgReport, MsgPushReportLog, &rsp)
+			return err
-	case MsgPushEditApp:
+		}
-		gLog.Println(LvINFO, "MsgPushEditApp")
+		gLog.i("SpecTunnel %d", req.TunnelIndex)
-		newApp := AppInfo{}
+		gConf.Network.specTunnel = int(req.TunnelIndex)
-		err := json.Unmarshal(msg[openP2PHeaderSize:], &newApp)
+	case MsgPushSDWanRefresh:
-		if err != nil {
+		GNetwork.write(MsgSDWAN, MsgSDWANInfoReq, nil)
-			gLog.Printf(LvERROR, "wrong MsgPushEditApp:%s  %s", err, string(msg[openP2PHeaderSize:]))
+	case MsgPushNat4Detect:
-			return err
+		handleNat4Detect(msg)
-		}
+	default:
-		oldConf := AppConfig{Enabled: 1}
+		i, ok := GNetwork.msgMap.Load(pushHead.From)
-		// protocol0+srcPort0 exist, delApp
+		if !ok {
-		oldConf.AppName = newApp.AppName
+			return ErrMsgChannelNotFound
-		oldConf.Protocol = newApp.Protocol0
+		}
-		oldConf.SrcPort = newApp.SrcPort0
+		ch := i.(chan msgCtx)
-		oldConf.PeerNode = newApp.PeerNode
+		ch <- msgCtx{data: msg, ts: time.Now()}
-		oldConf.DstHost = newApp.DstHost
+	}
-		oldConf.DstPort = newApp.DstPort
+	return err
-
+}
-		gConf.delete(oldConf)
+
-		// AddApp
+func handleNat4Detect(msg []byte) (err error) {
-		newConf := oldConf
+	gLog.d("handleNat4Detect")
-		newConf.Protocol = newApp.Protocol
+	nd := Nat4Detect{}
-		newConf.SrcPort = newApp.SrcPort
+	if err = json.Unmarshal(msg[openP2PHeaderSize:], &nd); err != nil {
-		gConf.add(newConf, false)
+		gLog.e("Unmarshal %v:%s  %s", reflect.TypeOf(nd), err, string(msg[openP2PHeaderSize:]))
-		gConf.save()          // save quickly for the next request reportApplist
+		return err
-		pn.DeleteApp(oldConf) // DeleteApp may cost some times, execute at the end
+	}
-		// autoReconnect will auto AddApp
+	detectNatPort := func(protocol, server string, serverPort, localPort int) int {
-		// pn.AddApp(config)
+		natPort := 0
-		// TODO: report result
+		if protocol == "tcp" {
-	case MsgPushEditNode:
+			_, natPort, _, _ = natDetectTCP(server, serverPort, localPort)
-		gLog.Println(LvINFO, "MsgPushEditNode")
+		} else {
-		req := EditNode{}
+			_, natPort, _ = natDetectUDP(server, serverPort, localPort)
-		err := json.Unmarshal(msg[openP2PHeaderSize:], &req)
+		}
-		if err != nil {
+		// gLog.i("%s %s %d %d %d", protocol, server, serverPort, localPort, natPort)
-			gLog.Printf(LvERROR, "wrong MsgPushEditNode:%s  %s", err, string(msg[openP2PHeaderSize:]))
+		return natPort
-			return err
+	}
-		}
+
-		gConf.setNode(req.NewName)
+	result := ""
-		gConf.setShareBandwidth(req.Bandwidth)
+	if nd.Num > 0 {
-		gConf.save()
+		for i := 0; i < int(nd.Num); i++ {
-		// TODO: hot reload
+			natPort := detectNatPort(nd.Protocol, nd.Server, int(nd.ServerPort), int(nd.LocalPort)+i)
-		os.Exit(0)
+			if i > 0 {
-	case MsgPushSwitchApp:
+				result += ","
-		gLog.Println(LvINFO, "MsgPushSwitchApp")
+			}
-		app := AppInfo{}
+			result += fmt.Sprintf("%d", natPort)
-		err := json.Unmarshal(msg[openP2PHeaderSize:], &app)
+		}
-		if err != nil {
+	} else {
-			gLog.Printf(LvERROR, "wrong MsgPushSwitchApp:%s  %s", err, string(msg[openP2PHeaderSize:]))
+		for idx, item := range nd.CustomData {
-			return err
+			natPort := detectNatPort(item.Protocol, item.Server, int(item.ServerPort), int(item.LocalPort))
-		}
+			if idx > 0 {
-		config := AppConfig{Enabled: app.Enabled, SrcPort: app.SrcPort, Protocol: app.Protocol}
+				result += ","
-		gLog.Println(LvINFO, app.AppName, " switch to ", app.Enabled)
+			}
-		gConf.switchApp(config, app.Enabled)
+			result += fmt.Sprintf("%d", natPort)
-		if app.Enabled == 0 {
+		}
-			// disable APP
+	}
-			pn.DeleteApp(config)
+	return GNetwork.write(MsgReport, MsgPushReportLog, &result)
-		}
+}
-	default:
+
-		pn.msgMapMtx.Lock()
+func handleEditApp(msg []byte) (err error) {
-		ch := pn.msgMap[pushHead.From]
+	gLog.i("MsgPushEditApp")
-		pn.msgMapMtx.Unlock()
+	newApp := AppInfo{}
-		ch <- msg
+	if err = json.Unmarshal(msg[openP2PHeaderSize:], &newApp); err != nil {
-	}
+		gLog.e("Unmarshal %v:%s  %s", reflect.TypeOf(newApp), err, string(msg[openP2PHeaderSize:]))
-	return nil
+		return err
-}
+	}
+	oldConf := AppConfig{Enabled: 1}
+	// protocol0+srcPort0 exist, delApp
+	oldConf.AppName = newApp.AppName
+	oldConf.Protocol = newApp.Protocol0
+	oldConf.Whitelist = newApp.Whitelist
+	oldConf.SrcPort = newApp.SrcPort0
+	oldConf.PeerNode = newApp.PeerNode
+	oldConf.DstHost = newApp.DstHost
+	oldConf.DstPort = newApp.DstPort
+	if newApp.Protocol0 != "" && newApp.SrcPort0 != 0 { // not edit
+		gConf.delete(oldConf)
+	}
+
+	if newApp.SrcPort != 0 { // delete app
+		// AddApp
+		newConf := oldConf
+		newConf.Protocol = newApp.Protocol
+		newConf.SrcPort = newApp.SrcPort
+		newConf.RelayNode = newApp.SpecRelayNode
+		newConf.PunchPriority = newApp.PunchPriority
+		gConf.add(newConf, false)
+	}
+
+	if newApp.Protocol0 != "" && newApp.SrcPort0 != 0 { // not edit
+		GNetwork.DeleteApp(oldConf) // DeleteApp may cost some times, execute at the end
+	}
+	return nil
+}
+
+func handleConnectReq(msg []byte) (err error) {
+	req := PushConnectReq{}
+	if err = json.Unmarshal(msg[openP2PHeaderSize+PushHeaderSize:], &req); err != nil {
+		gLog.e("Unmarshal %v:%s", reflect.TypeOf(req), err)
+		return err
+	}
+	gLog.d("%s is connecting... push connect response", req.From)
+	if compareVersion(req.Version, LeastSupportVersion) < 0 {
+		gLog.e("%s:%s", ErrVersionNotCompatible.Error(), req.From)
+		rsp := PushConnectRsp{
+			Error:  10,
+			Detail: ErrVersionNotCompatible.Error(),
+			To:     req.From,
+			From:   gConf.Network.Node,
+		}
+		GNetwork.push(req.From, MsgPushConnectRsp, rsp)
+		return ErrVersionNotCompatible
+	}
+	// verify totp token or token
+	t := totp.TOTP{Step: totp.RelayTOTPStep}
+	if t.Verify(req.Token, gConf.Network.Token, time.Now().Unix()-GNetwork.dt/int64(time.Second)) { // localTs may behind, auto adjust ts
+		gLog.d("handleConnectReq Access Granted")
+		config := AppConfig{}
+		config.peerNatType = req.NatType
+		config.peerConeNatPort = req.ConeNatPort
+		config.peerIP = req.FromIP
+		config.PeerNode = req.From
+		config.peerVersion = req.Version
+		config.fromToken = req.Token
+		config.peerIPv6 = req.IPv6
+		config.hasIPv4 = req.HasIPv4
+		config.hasUPNPorNATPMP = req.HasUPNPorNATPMP
+		config.linkMode = req.LinkMode
+		config.isUnderlayServer = req.IsUnderlayServer
+		config.UnderlayProtocol = req.UnderlayProtocol
+		// share relay node will limit bandwidth
+		if req.Token != gConf.Network.Token {
+			gLog.i("set share bandwidth %d mbps", gConf.Network.ShareBandwidth)
+			config.shareBandwidth = gConf.Network.ShareBandwidth
+		}
+		// go GNetwork.AddTunnel(config, req.ID)
+		go func() {
+			GNetwork.addDirectTunnel(config, req.ID, nil)
+		}()
+		return nil
+	}
+	gLog.e("handleConnectReq Access Denied:%s", req.From)
+	rsp := PushConnectRsp{
+		Error:  1,
+		Detail: fmt.Sprintf("connect to %s error: Access Denied", gConf.Network.Node),
+		To:     req.From,
+		From:   gConf.Network.Node,
+	}
+	return GNetwork.push(req.From, MsgPushConnectRsp, rsp)
+}
+
+func handleReportApps() (err error) {
+	gLog.i("MsgPushReportApps")
+	req := ReportApps{}
+	gConf.mtx.RLock()
+	defer gConf.mtx.RUnlock()
+
+	for _, config := range gConf.Apps {
+		appActive := 0
+		relayNode := ""
+		specRelayNode := ""
+		relayMode := ""
+		linkMode := LinkModeUDPPunch
+		var connectTime string
+		var retryTime string
+		app := GNetwork.findApp(config)
+		if app != nil {
+
+			if app.IsActive() {
+				appActive = 1
+			}
+			specRelayNode = app.config.RelayNode
+			t, tidx := app.AvailableTunnel()
+			if tidx != 0 { // TODO: should always report relay node for app edit
+				relayNode = app.relayNode[tidx]
+				relayMode = app.relayMode[tidx]
+			}
+
+			if t != nil {
+				linkMode = t.linkModeWeb
+			}
+			retryTime = app.RetryTime().Local().Format("2006-01-02T15:04:05-0700")
+			connectTime = app.ConnectTime().Local().Format("2006-01-02T15:04:05-0700")
+		}
+		appInfo := AppInfo{
+			AppName:       config.AppName,
+			Error:         config.errMsg,
+			Protocol:      config.Protocol,
+			PunchPriority: config.PunchPriority,
+			Whitelist:     config.Whitelist,
+			SrcPort:       config.SrcPort,
+			RelayNode:     relayNode,
+			SpecRelayNode: specRelayNode,
+			RelayMode:     relayMode,
+			LinkMode:      linkMode,
+			PeerNode:      config.PeerNode,
+			DstHost:       config.DstHost,
+			DstPort:       config.DstPort,
+			PeerUser:      config.PeerUser,
+			PeerIP:        config.peerIP,
+			PeerNatType:   config.peerNatType,
+			RetryTime:     retryTime,
+			ConnectTime:   connectTime,
+			IsActive:      appActive,
+			Enabled:       config.Enabled,
+		}
+		req.Apps = append(req.Apps, appInfo)
+	}
+	return GNetwork.write(MsgReport, MsgReportApps, &req)
+
+}
+
+func handleReportMemApps() (err error) {
+	gLog.i("handleReportMemApps")
+	req := ReportApps{}
+	GNetwork.sdwan.sysRoute.Range(func(key, value interface{}) bool {
+		node := value.(*sdwanNode)
+		appActive := 0
+		relayMode := ""
+		var connectTime string
+		var retryTime string
+
+		i, ok := GNetwork.apps.Load(node.id)
+		var app *p2pApp
+		var t *P2PTunnel
+		var tidx int
+		if ok {
+			app = i.(*p2pApp)
+			t, tidx = app.AvailableTunnel()
+			if app.IsActive() {
+				appActive = 1
+			}
+			if tidx != 0 {
+				relayMode = app.relayMode[tidx]
+			}
+			retryTime = app.RetryTime().Local().Format("2006-01-02T15:04:05-0700")
+			connectTime = app.ConnectTime().Local().Format("2006-01-02T15:04:05-0700")
+		}
+		appInfo := AppInfo{
+			RelayMode: relayMode,
+			PeerNode:  node.name,
+			IsActive:  appActive,
+			Enabled:   1,
+		}
+		if app != nil {
+			appInfo.AppName = app.config.AppName
+			appInfo.Error = app.config.errMsg
+			appInfo.Protocol = app.config.Protocol
+			appInfo.Whitelist = app.config.Whitelist
+			appInfo.SrcPort = app.config.SrcPort
+
+			if tidx != 0 {
+				appInfo.RelayNode = app.relayNode[tidx]
+			}
+
+			if t != nil {
+				appInfo.LinkMode = t.linkModeWeb
+			}
+			appInfo.DstHost = app.config.DstHost
+			appInfo.DstPort = app.config.DstPort
+			appInfo.PeerUser = app.config.PeerUser
+			appInfo.PeerIP = app.config.peerIP
+			appInfo.PeerNatType = app.config.peerNatType
+			appInfo.RetryTime = retryTime
+			appInfo.ConnectTime = connectTime
+		}
+		req.Apps = append(req.Apps, appInfo)
+		return true
+	})
+	req.TunError = GNetwork.sdwan.tunErr
+	gLog.d("handleReportMemApps res:%s", prettyJson(req))
+	gConf.retryAllMemApp()
+	return GNetwork.write(MsgReport, MsgReportMemApps, &req)
+}
+
+func handleLog(msg []byte) (err error) {
+	gLog.d("MsgPushReportLog")
+	const defaultLen = 1024 * 128
+	const maxLen = 1024 * 1024
+	req := ReportLogReq{}
+	if err = json.Unmarshal(msg[openP2PHeaderSize:], &req); err != nil {
+		gLog.e("Unmarshal %v:%s  %s", reflect.TypeOf(req), err, string(msg[openP2PHeaderSize:]))
+		return err
+	}
+	if req.FileName == "" {
+		req.FileName = "openp2p.log"
+	} else {
+		req.FileName = sanitizeFileName(req.FileName)
+	}
+	if req.IsSetLogLevel == 1 {
+		gLog.setLevel(LogLevel(req.LogLevel))
+	}
+	f, err := os.Open(filepath.Join("log", req.FileName))
+	if err != nil {
+		gLog.e("read log file error:%s", err)
+		return err
+	}
+	fi, err := f.Stat()
+	if err != nil {
+		return err
+	}
+	if req.Offset > fi.Size() {
+		req.Offset = fi.Size() - defaultLen
+	}
+	// verify input parameters
+	if req.Offset < 0 {
+		req.Offset = 0
+	}
+	if req.Len <= 0 || req.Len > maxLen {
+		req.Len = defaultLen
+	}
+
+	f.Seek(req.Offset, 0)
+	buff := make([]byte, req.Len)
+	readLength, err := f.Read(buff)
+	f.Close()
+	if err != nil {
+		gLog.e("read log content error:%s", err)
+		return err
+	}
+	rsp := ReportLogRsp{}
+	rsp.Content = string(buff[:readLength])
+	rsp.FileName = req.FileName
+	rsp.Total = fi.Size()
+	rsp.Len = req.Len
+	return GNetwork.write(MsgReport, MsgPushReportLog, &rsp)
+}
+
+func handleReportGoroutine() (err error) {
+	gLog.d("handleReportGoroutine")
+	buf := make([]byte, 1024*128)
+	stackLen := runtime.Stack(buf, true)
+	return GNetwork.write(MsgReport, MsgReportResponse, string(buf[:stackLen]))
+}
+
+func handleReportHeap() error {
+	var buf bytes.Buffer
+	err := pprof.Lookup("heap").WriteTo(&buf, 1)
+	if err != nil {
+		return err
+	}
+
+	return GNetwork.write(MsgReport, MsgReportResponse, buf.String())
+}
+
+func handleCheckRemoteService(msg []byte) (err error) {
+	gLog.d("handleCheckRemoteService")
+	req := CheckRemoteService{}
+	if err = json.Unmarshal(msg[openP2PHeaderSize:], &req); err != nil {
+		gLog.e("Unmarshal %v:%s  %s", reflect.TypeOf(req), err, string(msg[openP2PHeaderSize:]))
+		return err
+	}
+	rsp := PushRsp{Error: 0}
+	conn, err := net.DialTimeout("tcp", fmt.Sprintf("%s:%d", req.Host, req.Port), time.Second*3)
+	if err != nil {
+		rsp.Error = 1
+		rsp.Detail = ErrRemoteServiceUnable.Error()
+	} else {
+		conn.Close()
+	}
+	return GNetwork.write(MsgReport, MsgReportResponse, rsp)
+}

core/holepunch.go

@@ -3,182 +3,250 @@ package openp2p
 import (
 	"bytes"
 	"encoding/binary"
+	"encoding/json"
 	"fmt"
 	"math/rand"
 	"net"
-	"sync"
 	"time"
 )
 
 func handshakeC2C(t *P2PTunnel) (err error) {
-	gLog.Printf(LvDEBUG, "handshakeC2C %s:%d:%d to %s:%d", t.pn.config.Node, t.coneLocalPort, t.coneNatPort, t.config.peerIP, t.config.peerConeNatPort)
+	gLog.d("handshakeC2C %s:%d:%d to %s:%d", gConf.Network.Node, t.coneLocalPort, t.coneNatPort, t.config.peerIP, t.config.peerConeNatPort)
-	defer gLog.Printf(LvDEBUG, "handshakeC2C end")
+	defer gLog.d("handshakeC2C end")
-	conn, err := net.ListenUDP("udp", t.la)
+	conn, err := net.ListenUDP("udp", t.localHoleAddr)
 	if err != nil {
 		return err
 	}
 	defer conn.Close()
-	_, err = UDPWrite(conn, t.ra, MsgP2P, MsgPunchHandshake, P2PHandshakeReq{ID: t.id})
+	_, err = UDPWrite(conn, t.remoteHoleAddr, MsgP2P, MsgPunchHandshake, P2PHandshakeReq{ID: t.id})
 	if err != nil {
-		gLog.Println(LvDEBUG, "handshakeC2C write MsgPunchHandshake error:", err)
+		gLog.d("handshakeC2C write MsgPunchHandshake error:%s", err)
 		return err
 	}
-	ra, head, _, _, err := UDPRead(conn, 5000)
+	ra, head, buff, _, err := UDPRead(conn, HandshakeTimeout)
 	if err != nil {
-		time.Sleep(time.Millisecond * 200)
+		gLog.d("handshakeC2C read MsgPunchHandshake error:%s", err)
-		gLog.Println(LvDEBUG, err, ", return this error when ip was not reachable, retry read")
+		return err
-		ra, head, _, _, err = UDPRead(conn, 5000)
+	}
+	t.remoteHoleAddr, _ = net.ResolveUDPAddr("udp", ra.String())
+	var tunnelID uint64
+	if len(buff) > openP2PHeaderSize {
+		req := P2PHandshakeReq{}
+		if err := json.Unmarshal(buff[openP2PHeaderSize:openP2PHeaderSize+int(head.DataLen)], &req); err == nil {
+			tunnelID = req.ID
+		}
+	} else { // compatible with old version
+		tunnelID = t.id
+	}
+	if head.MainType == MsgP2P && head.SubType == MsgPunchHandshake && tunnelID == t.id {
+		gLog.d("read tunnelid:%d handshake ", t.id)
+		UDPWrite(conn, t.remoteHoleAddr, MsgP2P, MsgPunchHandshakeAck, P2PHandshakeReq{ID: t.id})
+		_, head, _, _, err = UDPRead(conn, HandshakeTimeout)
 		if err != nil {
-			gLog.Println(LvDEBUG, "handshakeC2C read MsgPunchHandshake error:", err)
+			gLog.d("handshakeC2C write MsgPunchHandshakeAck error:", err)
 			return err
 		}
 	}
-	t.ra, _ = net.ResolveUDPAddr("udp", ra.String())
+	if head.MainType == MsgP2P && head.SubType == MsgPunchHandshakeAck && tunnelID == t.id {
-	// cone server side
+		gLog.d("read tunnelID:%d handshake ack ", t.id)
-	if head.MainType == MsgP2P && head.SubType == MsgPunchHandshake {
+		_, err = UDPWrite(conn, t.remoteHoleAddr, MsgP2P, MsgPunchHandshakeAck, P2PHandshakeReq{ID: t.id})
-		gLog.Printf(LvDEBUG, "read %d handshake ", t.id)
-		UDPWrite(conn, t.ra, MsgP2P, MsgPunchHandshakeAck, P2PHandshakeReq{ID: t.id})
-		_, head, _, _, err = UDPRead(conn, 5000)
 		if err != nil {
-			gLog.Println(LvDEBUG, "handshakeC2C write MsgPunchHandshakeAck error", err)
+			gLog.d("handshakeC2C write MsgPunchHandshakeAck error:%s", err)
-			return err
-		}
-		if head.MainType == MsgP2P && head.SubType == MsgPunchHandshakeAck {
-			gLog.Printf(LvDEBUG, "read %d handshake ack ", t.id)
-			gLog.Printf(LvINFO, "handshakeC2C ok")
-			return nil
-		}
-	}
-	// cone client side will only read handshake ack
-	if head.MainType == MsgP2P && head.SubType == MsgPunchHandshakeAck {
-		gLog.Printf(LvDEBUG, "read %d handshake ack ", t.id)
-		_, err = UDPWrite(conn, t.ra, MsgP2P, MsgPunchHandshakeAck, P2PHandshakeReq{ID: t.id})
-		if err != nil {
-			gLog.Println(LvDEBUG, "handshakeC2C write MsgPunchHandshakeAck error", err)
 			return err
 		}
 	}
-	gLog.Printf(LvINFO, "handshakeC2C ok")
+	gLog.i("handshakeC2C ok")
 	return nil
 }
 
 func handshakeC2S(t *P2PTunnel) error {
-	gLog.Printf(LvDEBUG, "handshakeC2S start")
+	gLog.d("tid:%d handshakeC2S start", t.id)
-	defer gLog.Printf(LvDEBUG, "handshakeC2S end")
+	defer gLog.d("tid:%d handshakeC2S end", t.id)
-	// even if read timeout, continue handshake
+	if !buildTunnelMtx.TryLock() {
-	t.pn.read(t.config.PeerNode, MsgPush, MsgPushHandshakeStart, SymmetricHandshakeAckTimeout)
+		// time.Sleep(time.Second * 3)
+		return ErrBuildTunnelBusy
+	}
+	defer buildTunnelMtx.Unlock()
+	startTime := time.Now()
 	r := rand.New(rand.NewSource(time.Now().UnixNano()))
 	randPorts := r.Perm(65532)
-	conn, err := net.ListenUDP("udp", t.la)
+	conn, err := net.ListenUDP("udp", t.localHoleAddr)
 	if err != nil {
 		return err
 	}
 	defer conn.Close()
+
 	go func() error {
-		gLog.Printf(LvDEBUG, "send symmetric handshake to %s from %d:%d start", t.config.peerIP, t.coneLocalPort, t.coneNatPort)
+		gLog.d("tid:%d send symmetric handshake to %s from %d:%d start", t.id, t.config.peerIP, t.coneLocalPort, t.coneNatPort)
 		for i := 0; i < SymmetricHandshakeNum; i++ {
-			// TODO: auto calc cost time
+			// time.Sleep(SymmetricHandshakeInterval)
-			time.Sleep(SymmetricHandshakeInterval)
 			dst, err := net.ResolveUDPAddr("udp", fmt.Sprintf("%s:%d", t.config.peerIP, randPorts[i]+2))
 			if err != nil {
 				return err
 			}
 			_, err = UDPWrite(conn, dst, MsgP2P, MsgPunchHandshake, P2PHandshakeReq{ID: t.id})
 			if err != nil {
-				gLog.Println(LvDEBUG, "handshakeC2S write MsgPunchHandshake error:", err)
+				gLog.d("tid:%d handshakeC2S write MsgPunchHandshake error:%s", t.id, err)
 				return err
 			}
 		}
-		gLog.Println(LvDEBUG, "send symmetric handshake end")
+		gLog.d("tid:%d send symmetric handshake end", t.id)
 		return nil
 	}()
-	deadline := time.Now().Add(SymmetricHandshakeAckTimeout)
+	err = conn.SetReadDeadline(time.Now().Add(HandshakeTimeout))
-	err = conn.SetReadDeadline(deadline)
 	if err != nil {
-		gLog.Println(LvERROR, "SymmetricHandshakeAckTimeout SetReadDeadline error")
+		gLog.d("tid:%d SymmetricHandshakeAckTimeout SetReadDeadline error", t.id)
 		return err
 	}
 	// read response of the punching hole ok port
-	result := make([]byte, 1024)
+	buff := make([]byte, 1024)
-	_, dst, err := conn.ReadFrom(result)
+	_, dst, err := conn.ReadFrom(buff)
 	if err != nil {
-		gLog.Println(LvERROR, "handshakeC2S wait timeout")
+		gLog.d("tid:%d handshakeC2S wait timeout", t.id)
 		return err
 	}
 	head := &openP2PHeader{}
-	err = binary.Read(bytes.NewReader(result[:openP2PHeaderSize]), binary.LittleEndian, head)
+	err = binary.Read(bytes.NewReader(buff[:openP2PHeaderSize]), binary.LittleEndian, head)
 	if err != nil {
-		gLog.Println(LvERROR, "parse p2pheader error:", err)
+		gLog.e("tid:%d parse p2pheader error:%s", t.id, err)
 		return err
 	}
-	t.ra, _ = net.ResolveUDPAddr("udp", dst.String())
+	t.remoteHoleAddr, _ = net.ResolveUDPAddr("udp", dst.String())
+	var tunnelID uint64
+	if len(buff) > openP2PHeaderSize {
+		req := P2PHandshakeReq{}
+		if err := json.Unmarshal(buff[openP2PHeaderSize:openP2PHeaderSize+int(head.DataLen)], &req); err == nil {
+			tunnelID = req.ID
+		}
+	} else { // compatible with old version
+		tunnelID = t.id
+	}
+	if head.MainType == MsgP2P && head.SubType == MsgPunchHandshake && tunnelID == t.id {
+		gLog.d("tid:%d handshakeC2S read handshake ", t.id)
+		UDPWrite(conn, t.remoteHoleAddr, MsgP2P, MsgPunchHandshakeAck, P2PHandshakeReq{ID: t.id})
+		for {
+			_, head, buff, _, err = UDPRead(conn, HandshakeTimeout)
+			if err != nil {
+				gLog.d("tid:%d handshakeC2S handshake error", t.id)
+				return err
+			}
+			var tunnelID uint64
+			if len(buff) > openP2PHeaderSize {
+				req := P2PHandshakeReq{}
+				if err := json.Unmarshal(buff[openP2PHeaderSize:openP2PHeaderSize+int(head.DataLen)], &req); err == nil {
+					tunnelID = req.ID
+				}
+			} else { // compatible with old version
+				tunnelID = t.id
+			}
+			// waiting ack
+			if head.MainType == MsgP2P && head.SubType == MsgPunchHandshakeAck && tunnelID == t.id {
+				break
+			}
+		}
+	}
 	if head.MainType == MsgP2P && head.SubType == MsgPunchHandshakeAck {
-		gLog.Printf(LvDEBUG, "handshakeC2S read %d handshake ack %s", t.id, dst.String())
+		gLog.d("tid:%d handshakeC2S read handshake ack %s", t.id, t.remoteHoleAddr.String())
-		_, err = UDPWrite(conn, dst, MsgP2P, MsgPunchHandshakeAck, P2PHandshakeReq{ID: t.id})
+		_, err = UDPWrite(conn, t.remoteHoleAddr, MsgP2P, MsgPunchHandshakeAck, P2PHandshakeReq{ID: t.id})
 		return err
+	} else {
+		gLog.d("tid:%d handshakeS2C read msg but not MsgPunchHandshakeAck", t.id)
 	}
-	gLog.Printf(LvINFO, "handshakeC2S ok")
+	gLog.i("tid:%d handshakeC2S ok. cost %d ms", t.id, time.Since(startTime)/time.Millisecond)
 	return nil
 }
 
 func handshakeS2C(t *P2PTunnel) error {
-	gLog.Printf(LvDEBUG, "handshakeS2C start")
+	gLog.d("tid:%d handshakeS2C start", t.id)
-	defer gLog.Printf(LvDEBUG, "handshakeS2C end")
+	defer gLog.d("tid:%d handshakeS2C end", t.id)
-	gotCh := make(chan *net.UDPAddr, 5)
+	if !buildTunnelMtx.TryLock() {
+		// time.Sleep(time.Second * 3)
+		return ErrBuildTunnelBusy
+	}
+	defer buildTunnelMtx.Unlock()
+	startTime := time.Now()
+	gotCh := make(chan *net.UDPAddr, 50)
 	// sequencely udp send handshake, do not parallel send
-	gLog.Printf(LvDEBUG, "send symmetric handshake to %s:%d start", t.config.peerIP, t.config.peerConeNatPort)
+	gLog.d("tid:%d send symmetric handshake to %s:%d start", t.id, t.config.peerIP, t.config.peerConeNatPort)
 	gotIt := false
-	gotMtx := sync.Mutex{}
 	for i := 0; i < SymmetricHandshakeNum; i++ {
-		// TODO: auto calc cost time
+		// time.Sleep(SymmetricHandshakeInterval)
-		time.Sleep(SymmetricHandshakeInterval)
 		go func(t *P2PTunnel) error {
-			conn, err := net.ListenUDP("udp", nil)
+			conn, err := net.ListenUDP("udp", nil) // TODO: system allocated port really random?
 			if err != nil {
-				gLog.Printf(LvDEBUG, "listen error")
+				gLog.d("tid:%d listen error", t.id)
 				return err
 			}
 			defer conn.Close()
-			UDPWrite(conn, t.ra, MsgP2P, MsgPunchHandshake, P2PHandshakeReq{ID: t.id})
+			UDPWrite(conn, t.remoteHoleAddr, MsgP2P, MsgPunchHandshake, P2PHandshakeReq{ID: t.id})
-			_, head, _, _, err := UDPRead(conn, 10000)
+			_, head, buff, _, err := UDPRead(conn, HandshakeTimeout)
 			if err != nil {
 				// gLog.Println(LevelDEBUG, "one of the handshake error:", err)
 				return err
 			}
-			gotMtx.Lock()
-			defer gotMtx.Unlock()
 			if gotIt {
 				return nil
 			}
-			gotIt = true
+			var tunnelID uint64
-			t.la, _ = net.ResolveUDPAddr("udp", conn.LocalAddr().String())
+			if len(buff) >= openP2PHeaderSize+8 {
-			if head.MainType == MsgP2P && head.SubType == MsgPunchHandshake {
+				req := P2PHandshakeReq{}
-				gLog.Printf(LvDEBUG, "handshakeS2C read %d handshake ", t.id)
+				if err := json.Unmarshal(buff[openP2PHeaderSize:openP2PHeaderSize+int(head.DataLen)], &req); err == nil {
-				UDPWrite(conn, t.ra, MsgP2P, MsgPunchHandshakeAck, P2PHandshakeReq{ID: t.id})
+					tunnelID = req.ID
-				_, head, _, _, err = UDPRead(conn, 5000)
-				if err != nil {
-					gLog.Println(LvDEBUG, "handshakeS2C handshake error")
-					return err
 				}
-				if head.MainType == MsgP2P && head.SubType == MsgPunchHandshakeAck {
+			} else { // compatible with old version
-					gLog.Printf(LvDEBUG, "handshakeS2C read %d handshake ack %s", t.id, conn.LocalAddr().String())
+				tunnelID = t.id
-					gotCh <- t.la
+			}
-					return nil
+
+			if head.MainType == MsgP2P && head.SubType == MsgPunchHandshake && tunnelID == t.id {
+				gLog.d("tid:%d handshakeS2C read handshake ", t.id)
+				UDPWrite(conn, t.remoteHoleAddr, MsgP2P, MsgPunchHandshakeAck, P2PHandshakeReq{ID: t.id})
+				// may read several MsgPunchHandshake
+				for {
+					_, head, buff, _, err = UDPRead(conn, HandshakeTimeout)
+					if err != nil {
+						gLog.d("tid:%d handshakeS2C handshake error", t.id)
+						return err
+					}
+					if len(buff) > openP2PHeaderSize {
+						req := P2PHandshakeReq{}
+						if err := json.Unmarshal(buff[openP2PHeaderSize:openP2PHeaderSize+int(head.DataLen)], &req); err == nil {
+							tunnelID = req.ID
+						}
+					} else { // compatible with old version
+						tunnelID = t.id
+					}
+					if head.MainType == MsgP2P && head.SubType == MsgPunchHandshakeAck && tunnelID == t.id {
+						break
+					} else {
+						gLog.d("tid:%d handshakeS2C read msg but not MsgPunchHandshakeAck", t.id)
+					}
 				}
 			}
+			if head.MainType == MsgP2P && head.SubType == MsgPunchHandshakeAck {
+				gLog.d("tid:%d handshakeS2C read handshake ack %s", t.id, conn.LocalAddr().String())
+				UDPWrite(conn, t.remoteHoleAddr, MsgP2P, MsgPunchHandshakeAck, P2PHandshakeReq{ID: t.id})
+				gotIt = true
+				la, _ := net.ResolveUDPAddr("udp", conn.LocalAddr().String())
+				gotCh <- la
+				return nil
+			} else {
+				gLog.d("tid:%d handshakeS2C read msg but not MsgPunchHandshakeAck", t.id)
+			}
 			return nil
 		}(t)
 	}
-	gLog.Printf(LvDEBUG, "send symmetric handshake end")
+	gLog.d("tid:%d send symmetric handshake end", t.id)
-	gLog.Println(LvDEBUG, "handshakeS2C ready, notify peer connect")
+	if compareVersion(t.config.peerVersion, SymmetricSimultaneouslySendVersion) < 0 { // compatible with old client
-	t.pn.push(t.config.PeerNode, MsgPushHandshakeStart, TunnelMsg{ID: t.id})
+		gLog.d("tid:%d handshakeS2C ready, notify peer connect", t.id)
+		GNetwork.push(t.config.PeerNode, MsgPushHandshakeStart, TunnelMsg{ID: t.id})
+	}
 
 	select {
-	case <-time.After(SymmetricHandshakeAckTimeout):
+	case <-time.After(HandshakeTimeout):
-		return fmt.Errorf("wait handshake failed")
+		return fmt.Errorf("tid:%d wait handshake timeout", t.id)
 	case la := <-gotCh:
-		gLog.Println(LvDEBUG, "symmetric handshake ok", la)
+		t.localHoleAddr = la
-		gLog.Printf(LvINFO, "handshakeS2C ok")
+		gLog.i("tid:%d handshakeS2C ok. cost %dms", t.id, time.Since(startTime)/time.Millisecond)
 	}
 	return nil
 }

core/install.go

@@ -1,129 +1,123 @@
-package openp2p
+package openp2p
-
+
-import (
+import (
-	"fmt"
+	"fmt"
-	"io"
+	"io"
-	"os"
+	"os"
-	"os/exec"
+	"os/exec"
-	"path/filepath"
+	"path/filepath"
-	"strings"
+	"strings"
-	"time"
+	"time"
-)
+)
-
+
-// examples:
+func install() {
-// listen:
+	gLog.i("openp2p start. version: %s", OpenP2PVersion)
-// ./openp2p install -node hhd1207-222 -token YOUR-TOKEN -sharebandwidth 0
+	gLog.i("Contact: QQ group 16947733, Email [email protected]")
-// listen and build p2papp:
+	gLog.i("install start")
-// ./openp2p install -node hhd1207-222 -token YOUR-TOKEN -sharebandwidth 0 -peernode hhdhome-n1 -dstip 127.0.0.1 -dstport 50022 -protocol tcp -srcport 22
+	defer gLog.i("install end")
-func install() {
+	parseParams("install", "")
-	gLog.Println(LvINFO, "openp2p start. version: ", OpenP2PVersion)
+	// auto uninstall
-	gLog.Println(LvINFO, "Contact: QQ group 16947733, Email [email protected]")
+	uninstall(false)
-	gLog.Println(LvINFO, "install start")
+	gLog.i("install path: %s", defaultInstallPath)
-	defer gLog.Println(LvINFO, "install end")
+	targetPath := filepath.Join(defaultInstallPath, defaultBinName)
-	// auto uninstall
+	d := daemon{}
-	err := os.MkdirAll(defaultInstallPath, 0775)
+	// copy files
-
+
-	if err != nil {
+	binPath, _ := os.Executable()
-		gLog.Printf(LvERROR, "MkdirAll %s error:%s", defaultInstallPath, err)
+	src, errFiles := os.Open(binPath) // can not use args[0], on Windows call openp2p is ok(=openp2p.exe)
-		return
+	if errFiles != nil {
-	}
+		gLog.e("os.Open %s error:%s", os.Args[0], errFiles)
-	err = os.Chdir(defaultInstallPath)
+		return
-	if err != nil {
+	}
-		gLog.Println(LvERROR, "cd error:", err)
+
-		return
+	dst, errFiles := os.OpenFile(targetPath, os.O_CREATE|os.O_WRONLY|os.O_TRUNC, 0775)
-	}
+	if errFiles != nil {
-
+		time.Sleep(time.Second * 5) // maybe windows defender occupied the file, retry
-	uninstall()
+		dst, errFiles = os.OpenFile(targetPath, os.O_CREATE|os.O_WRONLY|os.O_TRUNC, 0775)
-	// save config file
+		if errFiles != nil {
-	parseParams("install")
+			gLog.e("os.OpenFile %s error:%s", targetPath, errFiles)
-	targetPath := filepath.Join(defaultInstallPath, defaultBinName)
+			return
-	d := daemon{}
+		}
-	// copy files
+	}
-
+
-	binPath, _ := os.Executable()
+	_, errFiles = io.Copy(dst, src)
-	src, errFiles := os.Open(binPath) // can not use args[0], on Windows call openp2p is ok(=openp2p.exe)
+	if errFiles != nil {
-	if errFiles != nil {
+		gLog.e("io.Copy error:%s", errFiles)
-		gLog.Printf(LvERROR, "os.OpenFile %s error:%s", os.Args[0], errFiles)
+		return
-		return
+	}
-	}
+	src.Close()
-
+	dst.Close()
-	dst, errFiles := os.OpenFile(targetPath, os.O_CREATE|os.O_WRONLY|os.O_TRUNC, 0775)
+
-	if errFiles != nil {
+	// install system service
-		gLog.Printf(LvERROR, "os.OpenFile %s error:%s", targetPath, errFiles)
+	err := d.Control("install", targetPath, []string{"-d"})
-		return
+	if err == nil {
-	}
+		gLog.i("install system service ok.")
-
+	}
-	_, errFiles = io.Copy(dst, src)
+	time.Sleep(time.Second * 2)
-	if errFiles != nil {
+	err = d.Control("start", targetPath, []string{"-d"})
-		gLog.Printf(LvERROR, "io.Copy error:%s", errFiles)
+	if err != nil {
-		return
+		gLog.e("start openp2p service error:%s", err)
-	}
+	} else {
-	src.Close()
+		gLog.i("start openp2p service ok.")
-	dst.Close()
+	}
-
+	gConf.save()
-	// install system service
+	gLog.i("Visit WebUI on https://console.openp2p.cn")
-	gLog.Println(LvINFO, "targetPath:", targetPath)
+}
-	err = d.Control("install", targetPath, []string{"-d"})
+
-	if err == nil {
+func installByFilename() {
-		gLog.Println(LvINFO, "install system service ok.")
+	params := strings.Split(filepath.Base(os.Args[0]), "-")
-	}
+	if len(params) < 4 {
-	time.Sleep(time.Second * 2)
+		return
-	err = d.Control("start", targetPath, []string{"-d"})
+	}
-	if err != nil {
+	serverHost := params[1]
-		gLog.Println(LvERROR, "start openp2p service error:", err)
+	token := params[2]
-	} else {
+	gLog.i("install start")
-		gLog.Println(LvINFO, "start openp2p service ok.")
+	targetPath := os.Args[0]
-	}
+	args := []string{"install"}
-	gLog.Println(LvINFO, "Visit WebUI on https://console.openp2p.cn")
+	args = append(args, "-serverhost")
-}
+	args = append(args, serverHost)
-
+	args = append(args, "-token")
-func installByFilename() {
+	args = append(args, token)
-	params := strings.Split(filepath.Base(os.Args[0]), "-")
+	env := os.Environ()
-	if len(params) < 4 {
+	cmd := exec.Command(targetPath, args...)
-		return
+	cmd.Stdout = os.Stdout
-	}
+	cmd.Stderr = os.Stderr
-	serverHost := params[1]
+	cmd.Stdin = os.Stdin
-	token := params[2]
+	cmd.Env = env
-	gLog.Println(LvINFO, "install start")
+	err := cmd.Run()
-	targetPath := os.Args[0]
+	if err != nil {
-	args := []string{"install"}
+		gLog.e("install by filename, start process error:%s", err)
-	args = append(args, "-serverhost")
+		return
-	args = append(args, serverHost)
+	}
-	args = append(args, "-token")
+	gLog.i("install end")
-	args = append(args, token)
+	gLog.i("Visit WebUI on https://console.openp2p.cn")
-	env := os.Environ()
+	fmt.Println("Press the Any Key to exit")
-	cmd := exec.Command(targetPath, args...)
+	fmt.Scanln()
-	cmd.Stdout = os.Stdout
+	os.Exit(0)
-	cmd.Stderr = os.Stderr
+}
-	cmd.Stdin = os.Stdin
+
-	cmd.Env = env
+func uninstall(rmFiles bool) {
-	err := cmd.Run()
+	gLog.i("uninstall start")
-	if err != nil {
+	defer gLog.i("uninstall end")
-		gLog.Println(LvERROR, "install by filename, start process error:", err)
+	d := daemon{}
-		return
+	err := d.Control("stop", "", nil)
-	}
+	if err != nil { // service maybe not install
-	gLog.Println(LvINFO, "install end")
+		gLog.d("stop service error:%s", err)
-	gLog.Println(LvINFO, "Visit WebUI on https://console.openp2p.cn")
+	}
-	fmt.Println("Press the Any Key to exit")
+	err = d.Control("uninstall", "", nil)
-	fmt.Scanln()
+	if err != nil {
-	os.Exit(0)
+		gLog.d("uninstall system service error:%s", err)
-}
+	} else {
-func uninstall() {
+		gLog.i("uninstall system service ok.")
-	gLog.Println(LvINFO, "uninstall start")
+	}
-	defer gLog.Println(LvINFO, "uninstall end")
+	time.Sleep(time.Second * 3)
-	d := daemon{}
+	binPath := filepath.Join(defaultInstallPath, defaultBinName)
-	err := d.Control("stop", "", nil)
+	os.Remove(binPath + "0")
-	if err != nil { // service maybe not install
+	os.Remove(binPath)
-		return
+	if rmFiles {
-	}
+		if err := os.RemoveAll(defaultInstallPath); err != nil {
-	err = d.Control("uninstall", "", nil)
+			gLog.e("RemoveAll %s error:%s", defaultInstallPath, err)
-	if err != nil {
+		}
-		gLog.Println(LvERROR, "uninstall system service error:", err)
+	}
-	} else {
+
-		gLog.Println(LvINFO, "uninstall system service ok.")
+}
-	}
-	binPath := filepath.Join(defaultInstallPath, defaultBinName)
-	os.Remove(binPath + "0")
-	os.Remove(binPath)
-	// os.RemoveAll(defaultInstallPath)  // reserve config.json
-}

core/iptables.go

@@ -0,0 +1,74 @@
+package openp2p
+
+import (
+	"log"
+	"os/exec"
+	"runtime"
+)
+
+func allowTunForward() {
+	if runtime.GOOS != "linux" { // only support Linux
+		return
+	}
+	exec.Command("sh", "-c", `iptables -t filter -D FORWARD -i optun -j ACCEPT`).Run()
+	exec.Command("sh", "-c", `iptables -t filter -D FORWARD -o optun -j ACCEPT`).Run()
+	err := exec.Command("sh", "-c", `iptables -t filter -I FORWARD -i optun -j ACCEPT`).Run()
+	if err != nil {
+		log.Println("allow foward in error:", err)
+	}
+	err = exec.Command("sh", "-c", `iptables -t filter -I FORWARD -o optun -j ACCEPT`).Run()
+	if err != nil {
+		log.Println("allow foward out error:", err)
+	}
+}
+
+func clearSNATRule() {
+	if runtime.GOOS != "linux" {
+		return
+	}
+	execCommand("iptables", true, "-t", "nat", "-D", "POSTROUTING", "-j", "OPSDWAN")
+	execCommand("iptables", true, "-t", "nat", "-F", "OPSDWAN")
+	execCommand("iptables", true, "-t", "nat", "-X", "OPSDWAN")
+}
+
+func initSNATRule(localNet string) {
+	if runtime.GOOS != "linux" {
+		return
+	}
+	clearSNATRule()
+
+	err := execCommand("iptables", true, "-t", "nat", "-N", "OPSDWAN")
+	if err != nil {
+		log.Println("iptables new sdwan chain error:", err)
+		return
+	}
+	err = execCommand("iptables", true, "-t", "nat", "-A", "POSTROUTING", "-j", "OPSDWAN")
+	if err != nil {
+		log.Println("iptables append postrouting error:", err)
+		return
+	}
+	err = execCommand("iptables", true, "-t", "nat", "-A", "OPSDWAN",
+		"-o", "optun", "!", "-s", localNet, "-j", "MASQUERADE")
+	if err != nil {
+		log.Println("add optun snat error:", err)
+		return
+	}
+	err = execCommand("iptables", true, "-t", "nat", "-A", "OPSDWAN", "!", "-o", "optun",
+		"-s", localNet, "-j", "MASQUERADE")
+	if err != nil {
+		log.Println("add optun snat error:", err)
+		return
+	}
+}
+
+func addSNATRule(target string) {
+	if runtime.GOOS != "linux" {
+		return
+	}
+	err := execCommand("iptables", true, "-t", "nat", "-A", "OPSDWAN", "!", "-o", "optun",
+		"-s", target, "-j", "MASQUERADE")
+	if err != nil {
+		log.Println("iptables add optun snat error:", err)
+		return
+	}
+}

core/iptree.go

@@ -0,0 +1,179 @@
+package openp2p
+
+import (
+	"bytes"
+	"encoding/binary"
+	"fmt"
+	"log"
+	"net"
+	"strings"
+	"sync"
+
+	"github.com/emirpasic/gods/trees/avltree"
+	"github.com/emirpasic/gods/utils"
+)
+
+type IPTree struct {
+	tree    *avltree.Tree
+	treeMtx sync.RWMutex
+}
+type IPTreeValue struct {
+	maxIP uint32
+	v     interface{}
+}
+
+// TODO: deal interset
+func (iptree *IPTree) DelIntIP(minIP uint32, maxIP uint32) {
+	iptree.tree.Remove(minIP)
+}
+
+// add 120k cost 0.5s
+func (iptree *IPTree) AddIntIP(minIP uint32, maxIP uint32, v interface{}) bool {
+	if minIP > maxIP {
+		return false
+	}
+	iptree.treeMtx.Lock()
+	defer iptree.treeMtx.Unlock()
+	newMinIP := minIP
+	newMaxIP := maxIP
+	cur := iptree.tree.Root
+	for {
+		if cur == nil {
+			break
+		}
+		tv := cur.Value.(*IPTreeValue)
+		curMinIP := cur.Key.(uint32)
+
+		// newNode all in existNode, treat as inserted.
+		if newMinIP >= curMinIP && newMaxIP <= tv.maxIP {
+			return true
+		}
+		// has no interset
+		if newMinIP > tv.maxIP {
+			cur = cur.Children[1]
+			continue
+		}
+		if newMaxIP < curMinIP {
+			cur = cur.Children[0]
+			continue
+		}
+		// has interset, rm it and Add the new merged ip segment
+		iptree.tree.Remove(curMinIP)
+		if curMinIP < newMinIP {
+			newMinIP = curMinIP
+		}
+		if tv.maxIP > newMaxIP {
+			newMaxIP = tv.maxIP
+		}
+		cur = iptree.tree.Root
+	}
+	//  put in the tree
+	iptree.tree.Put(newMinIP, &IPTreeValue{newMaxIP, v})
+	return true
+}
+
+func (iptree *IPTree) Add(minIPStr string, maxIPStr string, v interface{}) bool {
+	var minIP, maxIP uint32
+	binary.Read(bytes.NewBuffer(net.ParseIP(minIPStr).To4()), binary.BigEndian, &minIP)
+	binary.Read(bytes.NewBuffer(net.ParseIP(maxIPStr).To4()), binary.BigEndian, &maxIP)
+	return iptree.AddIntIP(minIP, maxIP, v)
+}
+
+func (iptree *IPTree) Del(minIPStr string, maxIPStr string) {
+	var minIP, maxIP uint32
+	binary.Read(bytes.NewBuffer(net.ParseIP(minIPStr).To4()), binary.BigEndian, &minIP)
+	binary.Read(bytes.NewBuffer(net.ParseIP(maxIPStr).To4()), binary.BigEndian, &maxIP)
+	iptree.DelIntIP(minIP, maxIP)
+}
+
+func (iptree *IPTree) Contains(ipStr string) bool {
+	var ip uint32
+	binary.Read(bytes.NewBuffer(net.ParseIP(ipStr).To4()), binary.BigEndian, &ip)
+	_, ok := iptree.Load(ip)
+	return ok
+}
+
+func IsLocalhost(ipStr string) bool {
+	if ipStr == "localhost" || ipStr == "127.0.0.1" || ipStr == "::1" {
+		return true
+	}
+	return false
+}
+
+func (iptree *IPTree) Load(ip uint32) (interface{}, bool) {
+	iptree.treeMtx.RLock()
+	defer iptree.treeMtx.RUnlock()
+	if iptree.tree == nil {
+		return nil, false
+	}
+	n := iptree.tree.Root
+	for n != nil {
+		tv := n.Value.(*IPTreeValue)
+		curMinIP := n.Key.(uint32)
+		switch {
+		case ip >= curMinIP && ip <= tv.maxIP: // hit
+			return tv.v, true
+		case ip < curMinIP:
+			n = n.Children[0]
+		default:
+			n = n.Children[1]
+		}
+	}
+	return nil, false
+}
+
+func (iptree *IPTree) Size() int {
+	iptree.treeMtx.RLock()
+	defer iptree.treeMtx.RUnlock()
+	return iptree.tree.Size()
+}
+
+func (iptree *IPTree) Print() {
+	iptree.treeMtx.RLock()
+	defer iptree.treeMtx.RUnlock()
+	log.Println("size:", iptree.Size())
+	log.Println(iptree.tree.String())
+}
+
+func (iptree *IPTree) Clear() {
+	iptree.treeMtx.Lock()
+	defer iptree.treeMtx.Unlock()
+	iptree.tree.Clear()
+}
+
+// input format 127.0.0.1,192.168.1.0/24,10.1.1.30-10.1.1.50
+// 127.0.0.1
+// 192.168.1.0/24
+// 192.168.1.1-192.168.1.10
+func NewIPTree(ips string) *IPTree {
+	iptree := &IPTree{
+		tree: avltree.NewWith(utils.UInt32Comparator),
+	}
+	ipArr := strings.Split(ips, ",")
+	for _, ip := range ipArr {
+		if strings.Contains(ip, "/") { // x.x.x.x/24
+			_, ipNet, err := net.ParseCIDR(ip)
+			if err != nil {
+				fmt.Println("Error parsing CIDR:", err)
+				continue
+			}
+			minIP := ipNet.IP.Mask(ipNet.Mask).String()
+			maxIP := calculateMaxIP(ipNet).String()
+			iptree.Add(minIP, maxIP, nil)
+		} else if strings.Contains(ip, "-") { // x.x.x.x-y.y.y.y
+			minAndMax := strings.Split(ip, "-")
+			iptree.Add(minAndMax[0], minAndMax[1], nil)
+		} else { // single ip
+			iptree.Add(ip, ip, nil)
+		}
+	}
+	return iptree
+}
+func calculateMaxIP(ipNet *net.IPNet) net.IP {
+	maxIP := make(net.IP, len(ipNet.IP))
+	copy(maxIP, ipNet.IP)
+	for i := range maxIP {
+		maxIP[i] |= ^ipNet.Mask[i]
+	}
+	return maxIP
+}

core/iptree_test.go

@@ -0,0 +1,174 @@
+package openp2p
+
+import (
+	"bytes"
+	"encoding/binary"
+	"net"
+	"testing"
+	"time"
+)
+
+func wrapTestContains(t *testing.T, iptree *IPTree, ip string, result bool) {
+	if iptree.Contains(ip) == result {
+		// t.Logf("compare version %s %s ok\n", v1, v2)
+	} else {
+		t.Errorf("test %s fail\n", ip)
+	}
+}
+func wrapBenchmarkContains(t *testing.B, iptree *IPTree, ip string, result bool) {
+	if iptree.Contains(ip) == result {
+		// t.Logf("compare version %s %s ok\n", v1, v2)
+	} else {
+		t.Errorf("test %s fail\n", ip)
+	}
+}
+
+func TestAllInputFormat(t *testing.T) {
+	iptree := NewIPTree("219.137.185.70,127.0.0.1,127.0.0.0/8,192.168.1.0/24,192.168.3.100-192.168.3.255,192.168.100.0-192.168.200.255")
+	wrapTestContains(t, iptree, "127.0.0.1", true)
+	wrapTestContains(t, iptree, "127.0.0.2", true)
+	wrapTestContains(t, iptree, "127.1.1.1", true)
+	wrapTestContains(t, iptree, "219.137.185.70", true)
+	wrapTestContains(t, iptree, "219.137.185.71", false)
+	wrapTestContains(t, iptree, "192.168.1.2", true)
+	wrapTestContains(t, iptree, "192.168.2.2", false)
+	wrapTestContains(t, iptree, "192.168.3.1", false)
+	wrapTestContains(t, iptree, "192.168.3.100", true)
+	wrapTestContains(t, iptree, "192.168.3.255", true)
+	wrapTestContains(t, iptree, "192.168.150.1", true)
+	wrapTestContains(t, iptree, "192.168.250.1", false)
+}
+
+func TestSingleIP(t *testing.T) {
+	iptree := NewIPTree("")
+	iptree.Add("219.137.185.70", "219.137.185.70", nil)
+	wrapTestContains(t, iptree, "219.137.185.70", true)
+	wrapTestContains(t, iptree, "219.137.185.71", false)
+}
+
+func TestWrongSegment(t *testing.T) {
+	iptree := NewIPTree("")
+	inserted := iptree.Add("87.251.75.0", "82.251.75.255", nil)
+	if inserted {
+		t.Errorf("TestWrongSegment failed\n")
+	}
+}
+
+func TestSegment2(t *testing.T) {
+	iptree := NewIPTree("")
+	iptree.Clear()
+	iptree.Add("10.1.5.50", "10.1.5.100", nil)
+	iptree.Add("10.1.1.50", "10.1.1.100", nil)
+	iptree.Add("10.1.2.50", "10.1.2.100", nil)
+	iptree.Add("10.1.6.50", "10.1.6.100", nil)
+	iptree.Add("10.1.7.50", "10.1.7.100", nil)
+	iptree.Add("10.1.3.50", "10.1.3.100", nil)
+	iptree.Add("10.1.1.1", "10.1.1.10", nil)    // no interset
+	iptree.Add("10.1.1.200", "10.1.1.250", nil) // no interset
+	iptree.Print()
+
+	iptree.Add("10.1.1.80", "10.1.1.90", nil) // all in
+	iptree.Add("10.1.1.40", "10.1.1.60", nil) // interset
+	iptree.Print()
+	iptree.Add("10.1.1.90", "10.1.1.110", nil) // interset
+	iptree.Print()
+	t.Logf("ipTree size:%d\n", iptree.Size())
+	wrapTestContains(t, iptree, "10.1.1.40", true)
+	wrapTestContains(t, iptree, "10.1.5.50", true)
+	wrapTestContains(t, iptree, "10.1.6.50", true)
+	wrapTestContains(t, iptree, "10.1.7.50", true)
+	wrapTestContains(t, iptree, "10.1.2.50", true)
+	wrapTestContains(t, iptree, "10.1.3.50", true)
+	wrapTestContains(t, iptree, "10.1.1.60", true)
+	wrapTestContains(t, iptree, "10.1.1.90", true)
+	wrapTestContains(t, iptree, "10.1.1.110", true)
+	wrapTestContains(t, iptree, "10.1.1.250", true)
+	wrapTestContains(t, iptree, "10.1.2.60", true)
+	wrapTestContains(t, iptree, "10.1.100.30", false)
+	wrapTestContains(t, iptree, "10.1.200.30", false)
+
+	iptree.Add("10.0.0.0", "10.255.255.255", nil) // will merge all segment
+	iptree.Print()
+	if iptree.Size() != 1 {
+		t.Errorf("merge ip segment error\n")
+	}
+
+}
+
+func BenchmarkBuildipTree20k(t *testing.B) {
+	iptree := NewIPTree("")
+	iptree.Clear()
+	iptree.Add("10.1.5.50", "10.1.5.100", nil)
+	iptree.Add("10.1.1.50", "10.1.1.100", nil)
+	iptree.Add("10.1.2.50", "10.1.2.100", nil)
+	iptree.Add("10.1.6.50", "10.1.6.100", nil)
+	iptree.Add("10.1.7.50", "10.1.7.100", nil)
+	iptree.Add("10.1.3.50", "10.1.3.100", nil)
+	iptree.Add("10.1.1.1", "10.1.1.10", nil)    // no interset
+	iptree.Add("10.1.1.200", "10.1.1.250", nil) // no interset
+	iptree.Add("10.1.1.80", "10.1.1.90", nil)   // all in
+	iptree.Add("10.1.1.40", "10.1.1.60", nil)   // interset
+	iptree.Add("10.1.1.90", "10.1.1.110", nil)  // interset
+	var minIP uint32
+	binary.Read(bytes.NewBuffer(net.ParseIP("10.1.1.1").To4()), binary.BigEndian, &minIP)
+
+	// insert 10k block ip single
+	nodeNum := uint32(10000 * 1)
+	gap := uint32(10)
+	for i := minIP; i < minIP+nodeNum*gap; i += gap {
+		iptree.AddIntIP(i, i, nil)
+		// t.Logf("ipTree size:%d\n", iptree.Size())
+	}
+	binary.Read(bytes.NewBuffer(net.ParseIP("100.1.1.1").To4()), binary.BigEndian, &minIP)
+	// insert 100k block ip segment
+	for i := minIP; i < minIP+nodeNum*gap; i += gap {
+		iptree.AddIntIP(i, i+5, nil)
+	}
+	t.Logf("ipTree size:%d\n", iptree.Size())
+	iptree.Clear()
+	t.Logf("clear. ipTree size:%d\n", iptree.Size())
+}
+func BenchmarkQuery(t *testing.B) {
+	ts := time.Now()
+	iptree := NewIPTree("")
+	iptree.Clear()
+	iptree.Add("10.1.5.50", "10.1.5.100", nil)
+	iptree.Add("10.1.1.50", "10.1.1.100", nil)
+	iptree.Add("10.1.2.50", "10.1.2.100", nil)
+	iptree.Add("10.1.6.50", "10.1.6.100", nil)
+	iptree.Add("10.1.7.50", "10.1.7.100", nil)
+	iptree.Add("10.1.3.50", "10.1.3.100", nil)
+	iptree.Add("10.1.1.1", "10.1.1.10", nil)    // no interset
+	iptree.Add("10.1.1.200", "10.1.1.250", nil) // no interset
+	iptree.Add("10.1.1.80", "10.1.1.90", nil)   // all in
+	iptree.Add("10.1.1.40", "10.1.1.60", nil)   // interset
+	iptree.Add("10.1.1.90", "10.1.1.110", nil)  // interset
+	var minIP uint32
+	binary.Read(bytes.NewBuffer(net.ParseIP("10.1.1.1").To4()), binary.BigEndian, &minIP)
+
+	// insert 10k block ip single
+	nodeNum := uint32(10000 * 1000)
+	gap := uint32(10)
+	for i := minIP; i < minIP+nodeNum*gap; i += gap {
+		iptree.AddIntIP(i, i, nil)
+		// t.Logf("ipTree size:%d\n", iptree.Size())
+	}
+	binary.Read(bytes.NewBuffer(net.ParseIP("100.1.1.1").To4()), binary.BigEndian, &minIP)
+	// insert 100k block ip segment
+	for i := minIP; i < minIP+nodeNum*gap; i += gap {
+		iptree.AddIntIP(i, i+5, nil)
+	}
+	t.Logf("ipTree size:%d cost:%dms\n", iptree.Size(), time.Since(ts)/time.Millisecond)
+	ts = time.Now()
+	// t.ResetTimer()
+	queryNum := 100 * 10000
+	for i := 0; i < queryNum; i++ {
+		iptree.Load(minIP + uint32(i))
+		wrapBenchmarkContains(t, iptree, "10.1.5.55", true)
+		wrapBenchmarkContains(t, iptree, "10.1.1.1", true)
+		wrapBenchmarkContains(t, iptree, "10.1.5.200", false)
+		wrapBenchmarkContains(t, iptree, "200.1.1.1", false)
+	}
+	t.Logf("query num:%d cost:%dms\n", queryNum*4, time.Since(ts)/time.Millisecond)
+
+}

core/log.go

@@ -3,162 +3,197 @@ package openp2p
 import (
 	"log"
 	"os"
+	"path/filepath"
 	"runtime"
 	"sync"
+	"sync/atomic"
 	"time"
 )
 
-type LogLevel int
+type LogLevel int32
 
 var gLog *logger
 
 const (
-	LvDEBUG LogLevel = iota
+	LvDev   LogLevel = -1
-	LvINFO
+	LvDEBUG LogLevel = 0
-	LvWARN
+	LvINFO  LogLevel = 1
-	LvERROR
+	LvWARN  LogLevel = 2
+	LvERROR LogLevel = 3
 )
 
-var (
+const logFileNames string = ".log"
-	logFileNames map[LogLevel]string
-	loglevel     map[LogLevel]string
-)
-
-func init() {
-	logFileNames = make(map[LogLevel]string)
-	loglevel = make(map[LogLevel]string)
-	logFileNames[0] = ".log"
-	loglevel[LvDEBUG] = "DEBUG"
-	loglevel[LvINFO] = "INFO"
-	loglevel[LvWARN] = "WARN"
-	loglevel[LvERROR] = "ERROR"
 
+var loglevel = map[LogLevel]string{
+	LvDEBUG: "DEBUG",
+	LvINFO:  "INFO",
+	LvWARN:  "WARN",
+	LvERROR: "ERROR",
+	LvDev:   "Dev",
 }
 
 const (
-	LogFile = iota
+	LogFile    = 1
-	LogConsole
+	LogConsole = 1 << 1
-	LogFileAndConsole
 )
 
 type logger struct {
-	loggers    map[LogLevel]*log.Logger
+	logger           *log.Logger
-	files      map[LogLevel]*os.File
+	files            *os.File
-	level      LogLevel
+	level            atomic.Int32
-	logDir     string
+	logDir           string
-	mtx        *sync.Mutex
+	mtx              sync.Mutex
-	lineEnding string
+	lineEnding       string
-	pid        int
+	pid              int
-	maxLogSize int64
+	maxLogSize       atomic.Int64
-	mode       int
+	mode             int
+	stdLogger        *log.Logger
+	checkFileRunning bool
 }
 
 func NewLogger(path string, filePrefix string, level LogLevel, maxLogSize int64, mode int) *logger {
-	loggers := make(map[LogLevel]*log.Logger)
+	logdir := filepath.Join(path, "log")
-	logfiles := make(map[LogLevel]*os.File)
+	if err := os.MkdirAll(logdir, 0755); err != nil && mode&LogFile != 0 {
-	var (
+		return nil
-		logdir string
-	)
-	if path == "" {
-		logdir = "log/"
-	} else {
-		logdir = path + "/log/"
 	}
-	os.MkdirAll(logdir, 0777)
+	logFilePath := filepath.Join(logdir, filePrefix+logFileNames)
-	for lv := range logFileNames {
+	f, err := os.OpenFile(logFilePath, os.O_CREATE|os.O_WRONLY|os.O_APPEND, 0644)
-		logFilePath := logdir + filePrefix + logFileNames[lv]
+	if err != nil && mode&LogFile != 0 {
-		f, err := os.OpenFile(logFilePath, os.O_CREATE|os.O_WRONLY|os.O_APPEND, 0666)
+		log.Fatal(err)
-		if err != nil {
-			log.Fatal(err)
-		}
-		os.Chmod(logFilePath, 0666)
-		logfiles[lv] = f
-		loggers[lv] = log.New(f, "", log.LstdFlags)
 	}
-	var le string
+	stdLog := log.New(f, "", log.LstdFlags|log.Lmicroseconds)
+	le := "\n"
 	if runtime.GOOS == "windows" {
 		le = "\r\n"
-	} else {
-		le = "\n"
 	}
-	pLog := &logger{loggers, logfiles, level, logdir, &sync.Mutex{}, le, os.Getpid(), maxLogSize, mode}
+	pLog := &logger{logger: stdLog,
+		files:      f,
+		logDir:     logdir,
+		lineEnding: le,
+		pid:        os.Getpid(),
+		mode:       mode,
+		stdLogger:  log.New(os.Stdout, "", 0)}
+	pLog.setMaxSize(maxLogSize)
+	pLog.setLevel(level)
+	pLog.stdLogger.SetFlags(log.LstdFlags | log.Lmicroseconds)
 	go pLog.checkFile()
 	return pLog
 }
 
 func (l *logger) setLevel(level LogLevel) {
-	l.mtx.Lock()
+	l.level.Store(int32(level))
-	defer l.mtx.Unlock()
-	l.level = level
 }
+
+func (l *logger) setMaxSize(size int64) {
+	l.maxLogSize.Store(size)
+}
+
 func (l *logger) setMode(mode int) {
 	l.mtx.Lock()
 	defer l.mtx.Unlock()
 	l.mode = mode
 }
 
+func (l *logger) close() {
+	l.checkFileRunning = false
+	l.files.Close()
+}
+
 func (l *logger) checkFile() {
-	if l.maxLogSize <= 0 {
+	if l.maxLogSize.Load() <= 0 {
 		return
 	}
+	l.checkFileRunning = true
 	ticker := time.NewTicker(time.Minute)
-	for {
+	for l.checkFileRunning {
 		select {
 		case <-ticker.C:
-			l.mtx.Lock()
+			f, e := l.files.Stat()
-			for lv, logFile := range l.files {
+			if e != nil {
-				f, e := logFile.Stat()
+				continue
-				if e != nil {
-					continue
-				}
-				if f.Size() <= l.maxLogSize {
-					continue
-				}
-				logFile.Close()
-				fname := f.Name()
-				backupPath := l.logDir + fname + ".0"
-				os.Remove(backupPath)
-				os.Rename(l.logDir+fname, backupPath)
-				newFile, e := os.OpenFile(l.logDir+fname, os.O_CREATE|os.O_WRONLY|os.O_APPEND, 0666)
-				if e == nil {
-					l.loggers[lv].SetOutput(newFile)
-					l.files[lv] = newFile
-				}
 			}
-			l.mtx.Unlock()
+			if f.Size() <= l.maxLogSize.Load() {
+				continue
+			}
+			l.mtx.Lock()
+			l.files.Close()
+			fname := f.Name()
+			backupPath := filepath.Join(l.logDir, fname+".0")
+			err := os.Remove(backupPath)
+			if err != nil {
+				log.Println("remove openp2p.log0 error:", err)
+			}
+			if err = os.Rename(filepath.Join(l.logDir, fname), backupPath); err != nil {
+				log.Println("rename openp2p.log error:", err)
+			}
+			if newFile, e := os.OpenFile(filepath.Join(l.logDir, fname), os.O_CREATE|os.O_WRONLY|os.O_APPEND, 0644); e == nil {
+
+				l.logger.SetOutput(newFile)
+				l.files = newFile
+				l.mtx.Unlock()
+			}
+		case <-time.After(time.Second * 1):
 		}
+
 	}
 }
 
 func (l *logger) Printf(level LogLevel, format string, params ...interface{}) {
-	l.mtx.Lock()
+	if level < LogLevel(l.level.Load()) {
-	defer l.mtx.Unlock()
-	if level < l.level {
 		return
 	}
+	l.mtx.Lock()
+	defer l.mtx.Unlock()
+
 	pidAndLevel := []interface{}{l.pid, loglevel[level]}
 	params = append(pidAndLevel, params...)
-	if l.mode == LogFile || l.mode == LogFileAndConsole {
+	if l.mode&LogFile != 0 {
-		l.loggers[0].Printf("%d %s "+format+l.lineEnding, params...)
+		l.logger.Printf("%d %s "+format+l.lineEnding, params...)
 	}
-	if l.mode == LogConsole || l.mode == LogFileAndConsole {
+	if l.mode&LogConsole != 0 {
-		log.Printf("%d %s "+format+l.lineEnding, params...)
+		l.stdLogger.Printf("%d %s "+format+l.lineEnding, params...)
 	}
 }
 
 func (l *logger) Println(level LogLevel, params ...interface{}) {
-	l.mtx.Lock()
+	if level < LogLevel(l.level.Load()) {
-	defer l.mtx.Unlock()
-	if level < l.level {
 		return
 	}
+	l.mtx.Lock()
+	defer l.mtx.Unlock()
 	pidAndLevel := []interface{}{l.pid, " ", loglevel[level], " "}
 	params = append(pidAndLevel, params...)
 	params = append(params, l.lineEnding)
-	if l.mode == LogFile || l.mode == LogFileAndConsole {
+	if l.mode&LogFile != 0 {
-		l.loggers[0].Print(params...)
+		l.logger.Print(params...)
 	}
-	if l.mode == LogConsole || l.mode == LogFileAndConsole {
+	if l.mode&LogConsole != 0 {
-		log.Print(params...)
+		l.stdLogger.Print(params...)
 	}
 }
+
+func (l *logger) d(format string, params ...interface{}) {
+	l.Printf(LvDEBUG, format, params...)
+}
+
+func (l *logger) i(format string, params ...interface{}) {
+	l.Printf(LvINFO, format, params...)
+}
+
+func (l *logger) w(format string, params ...interface{}) {
+	l.Printf(LvWARN, format, params...)
+}
+
+func (l *logger) e(format string, params ...interface{}) {
+	l.Printf(LvERROR, format, params...)
+}
+
+func (l *logger) dev(format string, params ...interface{}) {
+	l.Printf(LvDev, format, params...)
+}
+
+func InitForUnitTest(lv LogLevel) {
+	baseDir := filepath.Dir(os.Args[0])
+	os.Chdir(baseDir) // for system service
+	gLog = NewLogger(baseDir, ProductName, lv, 1024*1024, LogFile|LogConsole)
+}

core/nat.go

@@ -1,192 +1,214 @@
-package openp2p
+package openp2p
-
+
-import (
+import (
-	"encoding/json"
+	"encoding/json"
-	"fmt"
+	"fmt"
-	"log"
+	"math/rand"
-	"math/rand"
+	"net"
-	"net"
+	"strconv"
-	"strconv"
+	"strings"
-	"strings"
+	"time"
-	"sync"
+
-	"time"
+	upnp "openp2p/pkg/upnp"
-
+
-	reuse "github.com/openp2p-cn/go-reuseport"
+	reuse "github.com/openp2p-cn/go-reuseport"
-)
+)
-
+
-func natTCP(serverHost string, serverPort int, localPort int) (publicIP string, publicPort int) {
+func natDetectTCP(serverHost string, serverPort int, lp int) (publicIP string, publicPort int, localPort int, err error) {
-	// dialer := &net.Dialer{
+	gLog.dev("natDetectTCP start")
-	// 	LocalAddr: &net.TCPAddr{
+	defer gLog.dev("natDetectTCP end")
-	// 		IP:   net.ParseIP("0.0.0.0"),
+	conn, err := reuse.DialTimeout("tcp4", fmt.Sprintf("0.0.0.0:%d", lp), fmt.Sprintf("%s:%d", serverHost, serverPort), NatDetectTimeout)
-	// 		Port: localPort,
+	if err != nil {
-	// 	},
+		err = fmt.Errorf("dial tcp4 %s:%d error: %w", serverHost, serverPort, err)
-	// }
+		return
-	conn, err := reuse.DialTimeout("tcp4", fmt.Sprintf("%s:%d", "0.0.0.0", localPort), fmt.Sprintf("%s:%d", serverHost, serverPort), time.Second*5)
+	}
-	// conn, err := net.Dial("tcp4", fmt.Sprintf("%s:%d", serverHost, serverPort))
+	defer conn.Close()
-	if err != nil {
+
-		fmt.Printf("Dial tcp4 %s:%d error:%s", serverHost, serverPort, err)
+	localAddr := conn.LocalAddr().(*net.TCPAddr)
-		return
+	localPort = localAddr.Port
-	}
+
-	defer conn.Close()
+	if _, err = conn.Write([]byte("1")); err != nil {
-	_, wrerr := conn.Write([]byte("1"))
+		err = fmt.Errorf("write error: %w", err)
-	if wrerr != nil {
+		return
-		fmt.Printf("Write error: %s\n", wrerr)
+	}
-		return
+
-	}
+	b := make([]byte, 1000)
-	b := make([]byte, 1000)
+	conn.SetReadDeadline(time.Now().Add(NatDetectTimeout))
-	conn.SetReadDeadline(time.Now().Add(time.Second * 5))
+	n, err := conn.Read(b)
-	n, rderr := conn.Read(b)
+	if err != nil {
-	if rderr != nil {
+		err = fmt.Errorf("read error: %w", err)
-		fmt.Printf("Read error: %s\n", rderr)
+		return
-		return
+	}
-	}
+
-	arr := strings.Split(string(b[:n]), ":")
+	response := strings.Split(string(b[:n]), ":")
-	if len(arr) < 2 {
+	if len(response) < 2 {
-		return
+		err = fmt.Errorf("invalid response format: %s", string(b[:n]))
-	}
+		return
-	publicIP = arr[0]
+	}
-	port, _ := strconv.ParseInt(arr[1], 10, 32)
+
-	publicPort = int(port)
+	publicIP = response[0]
-	return
+	port, err := strconv.Atoi(response[1])
-
+	if err != nil {
-}
+		err = fmt.Errorf("invalid port format: %w", err)
-func natTest(serverHost string, serverPort int, localPort int) (publicIP string, publicPort int, err error) {
+		return
-	gLog.Println(LvDEBUG, "natTest start")
+	}
-	defer gLog.Println(LvDEBUG, "natTest end")
+	publicPort = port
-	conn, err := net.ListenPacket("udp", fmt.Sprintf(":%d", localPort))
+
-	if err != nil {
+	return
-		gLog.Println(LvERROR, "natTest listen udp error:", err)
+}
-		return "", 0, err
+
-	}
+func natDetectUDP(serverHost string, serverPort int, localPort int) (publicIP string, publicPort int, err error) {
-	defer conn.Close()
+	gLog.dev("natDetectUDP start")
-
+	defer gLog.dev("natDetectUDP end")
-	dst, err := net.ResolveUDPAddr("udp", fmt.Sprintf("%s:%d", serverHost, serverPort))
+	conn, err := net.ListenPacket("udp", fmt.Sprintf(":%d", localPort))
-	if err != nil {
+	if err != nil {
-		return "", 0, err
+		gLog.e("natDetectUDP listen udp error:%s", err)
-	}
+		return "", 0, err
-
+	}
-	// The connection can write data to the desired address.
+	defer conn.Close()
-	msg, err := newMessage(MsgNATDetect, 0, nil)
+
-	_, err = conn.WriteTo(msg, dst)
+	dst, err := net.ResolveUDPAddr("udp", fmt.Sprintf("%s:%d", serverHost, serverPort))
-	if err != nil {
+	if err != nil {
-		return "", 0, err
+		return "", 0, err
-	}
+	}
-	deadline := time.Now().Add(NatTestTimeout)
+
-	err = conn.SetReadDeadline(deadline)
+	// The connection can write data to the desired address.
-	if err != nil {
+	msg, err := newMessage(MsgNATDetect, MsgNAT, nil)
-		return "", 0, err
+	_, err = conn.WriteTo(msg, dst)
-	}
+	if err != nil {
-	buffer := make([]byte, 1024)
+		return "", 0, err
-	nRead, _, err := conn.ReadFrom(buffer)
+	}
-	if err != nil {
+	deadline := time.Now().Add(NatDetectTimeout)
-		gLog.Println(LvERROR, "NAT detect error:", err)
+	err = conn.SetReadDeadline(deadline)
-		return "", 0, err
+	if err != nil {
-	}
+		return "", 0, err
-	natRsp := NatDetectRsp{}
+	}
-	err = json.Unmarshal(buffer[openP2PHeaderSize:nRead], &natRsp)
+	buffer := make([]byte, 1024)
-
+	nRead, _, err := conn.ReadFrom(buffer)
-	return natRsp.IP, natRsp.Port, nil
+	if err != nil {
-}
+		gLog.e("NAT detect error:%s", err)
-
+		return "", 0, err
-func getNATType(host string, udp1 int, udp2 int) (publicIP string, NATType int, hasIPvr int, hasUPNPorNATPMP int, err error) {
+	}
-	// the random local port may be used by other.
+	natRsp := NatDetectRsp{}
-	localPort := int(rand.Uint32()%15000 + 50000)
+	json.Unmarshal(buffer[openP2PHeaderSize:nRead], &natRsp)
-	echoPort := P2PNetworkInstance(nil).config.TCPPort
+
-	ip1, port1, err := natTest(host, udp1, localPort)
+	return natRsp.IP, natRsp.Port, nil
-	if err != nil {
+}
-		return "", 0, 0, 0, err
+
-	}
+func getNATType(host string, detectPort1 int, detectPort2 int) (publicIP string, NATType int, err error) {
-	hasIPv4, hasUPNPorNATPMP := publicIPTest(ip1, echoPort)
+	setUPNP(gConf.Network.PublicIPPort)
-	gLog.Printf(LvINFO, "local port:%d, nat port:%d, hasIPv4:%d, UPNP:%d", localPort, port1, hasIPv4, hasUPNPorNATPMP)
+	// the random local port may be used by other.
-	_, port2, err := natTest(host, udp2, localPort) // 2rd nat test not need testing publicip
+	localPort := int(rand.Uint32()%15000 + 50000)
-	gLog.Printf(LvDEBUG, "local port:%d  nat port:%d", localPort, port2)
+
-	if err != nil {
+	ip1, port1, err := natDetectUDP(host, detectPort1, localPort)
-		return "", 0, hasIPv4, hasUPNPorNATPMP, err
+	if err != nil {
-	}
+		// udp block try tcp
-	natType := NATSymmetric
+		gLog.w("udp block, try tcp nat detect")
-	if port1 == port2 {
+		if ip1, port1, _, err = natDetectTCP(host, detectPort1, localPort); err != nil {
-		natType = NATCone
+			return "", 0, err
-	}
+		}
-	return ip1, natType, hasIPv4, hasUPNPorNATPMP, nil
+	}
-}
+	_, port2, err := natDetectUDP(host, detectPort2, localPort) // 2rd nat test not need testing publicip
-
+	if err != nil {
-func publicIPTest(publicIP string, echoPort int) (hasPublicIP int, hasUPNPorNATPMP int) {
+		gLog.w("udp block, try tcp nat detect")
-	var echoConn *net.UDPConn
+		if _, port2, _, err = natDetectTCP(host, detectPort2, localPort); err != nil {
-	var wg sync.WaitGroup
+			return "", 0, err
-	wg.Add(1)
+		}
-	go func() {
+	}
-		gLog.Println(LvDEBUG, "echo server start")
+	gLog.d("local port:%d  nat port:%d", localPort, port2)
-		var err error
+	natType := NATSymmetric
-		echoConn, err = net.ListenUDP("udp", &net.UDPAddr{IP: net.IPv4zero, Port: echoPort})
+	if port1 == port2 {
-		if err != nil {
+		natType = NATCone
-			gLog.Println(LvERROR, "echo server listen error:", err)
+	}
-			return
+	return ip1, natType, nil
-		}
+}
-		buf := make([]byte, 1600)
+
-		// close outside for breaking the ReadFromUDP
+func publicIPTest(publicIP string, echoPort int) (hasPublicIP int, hasUPNPorNATPMP int) {
-		// wait 5s for echo testing
+	if publicIP == "" || echoPort == 0 {
-		wg.Done()
+		return
-		echoConn.SetReadDeadline(time.Now().Add(time.Second * 30))
+	}
-		n, addr, err := echoConn.ReadFromUDP(buf)
+	var echoConn *net.UDPConn
-		if err != nil {
+	gLog.d("echo server start")
-			return
+	var err error
-		}
+	echoConn, err = net.ListenUDP("udp", &net.UDPAddr{IP: net.IPv4zero, Port: echoPort})
-		echoConn.WriteToUDP(buf[0:n], addr)
+	if err != nil { // listen error
-		gLog.Println(LvDEBUG, "echo server end")
+		gLog.e("echo server listen error:%s", err)
-	}()
+		return
-	wg.Wait() // wait echo udp
+	}
-	defer echoConn.Close()
+	defer echoConn.Close()
-	// testing for public ip
+	// testing for public ip
-	for i := 0; i < 2; i++ {
+	for i := 0; i < 2; i++ {
-		if i == 1 {
+		if i == 1 {
-			// test upnp or nat-pmp
+			// test upnp or nat-pmp
-			gLog.Println(LvDEBUG, "upnp test start")
+			gLog.d("upnp test start")
-			nat, err := Discover()
+			// 7 days for udp connection
-			if err != nil || nat == nil {
+			// 7 days for tcp connection
-				gLog.Println(LvDEBUG, "could not perform UPNP discover:", err)
+			setUPNP(echoPort)
-				break
+		}
-			}
+		gLog.d("public ip test start %s:%d", publicIP, echoPort)
-			ext, err := nat.GetExternalAddress()
+		conn, err := net.ListenUDP("udp", nil)
-			if err != nil {
+		if err != nil {
-				gLog.Println(LvDEBUG, "could not perform UPNP external address:", err)
+			break
-				break
+		}
-			}
+		defer conn.Close()
-			log.Println("PublicIP:", ext)
+		dst, err := net.ResolveUDPAddr("udp", fmt.Sprintf("%s:%d", gConf.Network.ServerIP, gConf.Network.ServerPort))
-
+		if err != nil {
-			externalPort, err := nat.AddPortMapping("udp", echoPort, echoPort, "openp2p", 30)
+			break
-			if err != nil {
+		}
-				gLog.Println(LvDEBUG, "could not add udp UPNP port mapping", externalPort)
+
-				break
+		// The connection can write data to the desired address.
-			} else {
+		msg, _ := newMessage(MsgNATDetect, MsgPublicIP, NatDetectReq{EchoPort: echoPort})
-				nat.AddPortMapping("tcp", echoPort, echoPort, "openp2p", 604800)
+		_, err = conn.WriteTo(msg, dst)
-			}
+		if err != nil {
-		}
+			continue
-		gLog.Printf(LvDEBUG, "public ip test start %s:%d", publicIP, echoPort)
+		}
-		conn, err := net.ListenUDP("udp", nil)
+		buf := make([]byte, 1600)
-		if err != nil {
+
-			break
+		// wait for echo testing
-		}
+		echoConn.SetReadDeadline(time.Now().Add(PublicIPEchoTimeout))
-		defer conn.Close()
+		nRead, _, err := echoConn.ReadFromUDP(buf)
-		dst, err := net.ResolveUDPAddr("udp", fmt.Sprintf("%s:%d", publicIP, echoPort))
+		if err != nil {
-		if err != nil {
+			gLog.d("publicIPTest echoConn read timeout:%s", err)
-			break
+			continue
-		}
+		}
-		conn.WriteTo([]byte("echo"), dst)
+		natRsp := NatDetectRsp{}
-		buf := make([]byte, 1600)
+		err = json.Unmarshal(buf[openP2PHeaderSize:nRead], &natRsp)
-
+		if err != nil {
-		// wait for echo testing
+			gLog.d("publicIPTest Unmarshal error:%s", err)
-		conn.SetReadDeadline(time.Now().Add(PublicIPEchoTimeout))
+			continue
-		_, _, err = conn.ReadFromUDP(buf)
+		}
-		if err == nil {
+		if natRsp.Port == echoPort {
-			if i == 1 {
+			if i == 1 {
-				gLog.Println(LvDEBUG, "UPNP or NAT-PMP:YES")
+				gLog.d("UPNP or NAT-PMP:YES")
-				hasUPNPorNATPMP = 1
+				hasUPNPorNATPMP = 1
-			} else {
+			} else {
-				gLog.Println(LvDEBUG, "public ip:YES")
+				gLog.d("public ip:YES")
-				hasPublicIP = 1
+				hasPublicIP = 1
-			}
+			}
-			break
+			break
-		}
+		}
-	}
+	}
-	return
+	return
-}
+}
+
+func setUPNP(echoPort int) {
+	nat := upnp.Any() // Initialize the NAT interface
+	if nat == nil {
+		gLog.d("NAT interface is not available")
+		return
+	}
+	ext, err := nat.ExternalIP()
+	if err != nil {
+		gLog.d("could not perform UPNP external address:%s", err)
+		return
+	}
+	gLog.i("PublicIP:%v", ext)
+
+	externalPort, err := nat.AddMapping("udp", echoPort, echoPort, "openp2p", 604800)
+	if err != nil {
+		gLog.d("could not add udp UPNP port mapping %d", externalPort)
+		return
+	} else {
+		nat.AddMapping("tcp", echoPort, echoPort, "openp2p", 604800)
+	}
+}

core/openp2p.go

@@ -1,100 +1,148 @@
-package openp2p
+package openp2p
-
+
-import (
+import (
-	"fmt"
+	"fmt"
-	"math/rand"
+	"log"
-	"os"
+	"math/rand"
-	"path/filepath"
+	"os"
-	"strconv"
+	"path/filepath"
-	"time"
+	"strconv"
-)
+	"time"
-
+)
-func Run() {
+
-	rand.Seed(time.Now().UnixNano())
+var GNetwork *P2PNetwork
-	baseDir := filepath.Dir(os.Args[0])
+
-	os.Chdir(baseDir) // for system service
+func Run() {
-	gLog = NewLogger(baseDir, ProducnName, LvDEBUG, 1024*1024, LogFileAndConsole)
+	rand.Seed(time.Now().UnixNano())
-	// TODO: install sub command, deamon process
+	baseDir := filepath.Dir(os.Args[0])
-	if len(os.Args) > 1 {
+	os.Chdir(baseDir) // for system service
-		switch os.Args[1] {
+	gLog = NewLogger(baseDir, ProductName, LvDEBUG, 1024*1024, LogFile|LogConsole)
-		case "version", "-v", "--version":
+	if len(os.Args) > 1 {
-			fmt.Println(OpenP2PVersion)
+		switch os.Args[1] {
-			return
+		case "version", "-v", "--version":
-		case "update":
+			fmt.Println(OpenP2PVersion)
-			gLog = NewLogger(baseDir, ProducnName, LvDEBUG, 1024*1024, LogFileAndConsole)
+			return
-			targetPath := filepath.Join(defaultInstallPath, defaultBinName)
+		case "install":
-			d := daemon{}
+			install()
-			err := d.Control("restart", targetPath, nil)
+			return
-			if err != nil {
+		case "uninstall":
-				gLog.Println(LvERROR, "restart service error:", err)
+			uninstall(true)
-			} else {
+			return
-				gLog.Println(LvINFO, "restart service ok.")
+		case "start":
-			}
+			d := daemon{}
-			return
+			err := d.Control("start", "", nil)
-		case "install":
+			if err != nil {
-			install()
+				log.Println("openp2p start error:", err)
-			return
+				return
-		case "uninstall":
+			}
-			uninstall()
+			log.Println("openp2p start ok")
-			return
+			return
-		}
+		case "stop":
-	} else {
+			d := daemon{}
-		installByFilename()
+			err := d.Control("stop", "", nil)
-	}
+			if err != nil {
-	parseParams("")
+				log.Println("openp2p stop error:", err)
-	gLog.Println(LvINFO, "openp2p start. version: ", OpenP2PVersion)
+				return
-	gLog.Println(LvINFO, "Contact: QQ group 16947733, Email [email protected]")
+			}
-
+			log.Println("openp2p stop ok")
-	if gConf.daemonMode {
+			return
-		d := daemon{}
+		}
-		d.run()
+	} else {
-		return
+		installByFilename()
-	}
+	}
-
+	parseParams("", "")
-	gLog.Println(LvINFO, &gConf)
+	gLog.i("openp2p start. version: %s", OpenP2PVersion)
-	setFirewall()
+	gLog.i("Contact: QQ group 16947733, Email [email protected]")
-	network := P2PNetworkInstance(&gConf.Network)
+
-	if ok := network.Connect(30000); !ok {
+	if gConf.daemonMode {
-		gLog.Println(LvERROR, "P2PNetwork login error")
+		d := daemon{}
-		return
+		d.run()
-	}
+		return
-	gLog.Println(LvINFO, "waiting for connection...")
+	}
-	forever := make(chan bool)
+
-	<-forever
+	gLog.i("node=%s, serverHost=%s, serverPort=%d", gConf.Network.Node, gConf.Network.ServerHost, gConf.Network.ServerPort)
-}
+	setFirewall()
-
+	err := setRLimit()
-var network *P2PNetwork
+	if err != nil {
-
+		gLog.i("setRLimit error:%s", err)
-// for Android app
+	}
-// gomobile not support uint64 exported to java
+	P2PNetworkInstance()
-func RunAsModule(baseDir string, token string, bw int, logLevel int) *P2PNetwork {
+	if ok := GNetwork.Connect(30000); !ok {
-	rand.Seed(time.Now().UnixNano())
+		gLog.e("P2PNetwork login error")
-	os.Chdir(baseDir) // for system service
+		return
-	gLog = NewLogger(baseDir, ProducnName, LvDEBUG, 1024*1024, LogFileAndConsole)
+	}
-
+	// gLog.i("waiting for connection...")
-	parseParams("")
+	forever := make(chan bool)
-
+	<-forever
-	n, err := strconv.ParseUint(token, 10, 64)
+}
-	if err == nil {
+
-		gConf.setToken(n)
+// for Android app
-	}
+// gomobile not support uint64 exported to java
-	gLog.setLevel(LogLevel(logLevel))
+
-	gConf.setShareBandwidth(bw)
+func RunAsModule(baseDir string, token string, bw int, logLevel int) *P2PNetwork {
-	gLog.Println(LvINFO, "openp2p start. version: ", OpenP2PVersion)
+	rand.Seed(time.Now().UnixNano())
-	gLog.Println(LvINFO, "Contact: QQ group 16947733, Email [email protected]")
+	os.Chdir(baseDir) // for system service
-	gLog.Println(LvINFO, &gConf)
+	gLog = NewLogger(baseDir, ProductName, LvINFO, 1024*1024, LogFile|LogConsole)
-
+
-	network = P2PNetworkInstance(&gConf.Network)
+	parseParams("", "")
-	if ok := network.Connect(30000); !ok {
+
-		gLog.Println(LvERROR, "P2PNetwork login error")
+	n, err := strconv.ParseUint(token, 10, 64)
-		return nil
+	if err == nil && n > 0 {
-	}
+		gConf.setToken(n)
-	gLog.Println(LvINFO, "waiting for connection...")
+	}
-	return network
+	if n <= 0 && gConf.Network.Token == 0 { // not input token
-}
+		return nil
-
+	}
-func GetToken(baseDir string) string {
+	// gLog.setLevel(LogLevel(logLevel))
-	os.Chdir(baseDir)
+	gConf.setShareBandwidth(bw)
-	gConf.load()
+	gLog.i("openp2p start. version: %s", OpenP2PVersion)
-	return fmt.Sprintf("%d", gConf.Network.Token)
+	gLog.i("Contact: QQ group 16947733, Email [email protected]")
-}
+	gLog.i("node=%s, serverHost=%s, serverPort=%d", gConf.Network.Node, gConf.Network.ServerHost, gConf.Network.ServerPort)
+
+	P2PNetworkInstance()
+	if ok := GNetwork.Connect(30000); !ok {
+		gLog.e("P2PNetwork login error")
+		return nil
+	}
+	// gLog.i("waiting for connection...")
+	return GNetwork
+}
+
+func RunCmd(cmd string) {
+	rand.Seed(time.Now().UnixNano())
+	baseDir := filepath.Dir(os.Args[0])
+	os.Chdir(baseDir) // for system service
+	gLog = NewLogger(baseDir, ProductName, LvINFO, 1024*1024, LogFile|LogConsole)
+
+	parseParams("", cmd)
+	setFirewall()
+	err := setRLimit()
+	if err != nil {
+		gLog.i("setRLimit error:%s", err)
+	}
+	P2PNetworkInstance()
+	if ok := GNetwork.Connect(30000); !ok {
+		gLog.e("P2PNetwork login error")
+		return
+	}
+	forever := make(chan bool)
+	<-forever
+}
+
+func GetToken(baseDir string) string {
+	os.Chdir(baseDir)
+	gConf.load()
+	return fmt.Sprintf("%d", gConf.Network.Token)
+}
+
+func SetToken(token string) {
+	n, err := strconv.ParseUint(token, 10, 64)
+	if err == nil && n > 0 {
+		gConf.setToken(n)
+		gConf.save()
+	}
+}
+
+func Stop() {
+	os.Exit(0)
+}

core/optun.go

@@ -0,0 +1,23 @@
+package openp2p
+
+import (
+	"github.com/openp2p-cn/wireguard-go/tun"
+)
+
+const optunMTU = 1420
+
+var AndroidSDWANConfig chan []byte
+var preAndroidSDWANConfig string
+
+type optun struct {
+	tunName string
+	dev     tun.Device
+}
+
+func (t *optun) Stop() error {
+	t.dev.Close()
+	return nil
+}
+func init() {
+	AndroidSDWANConfig = make(chan []byte, 1)
+}

core/optun_android.go

@@ -0,0 +1,95 @@
+// optun_android.go
+//go:build android
+// +build android
+
+package openp2p
+
+import (
+	"time"
+)
+
+const (
+	tunIfaceName    = "optun"
+	PIHeaderSize    = 0
+	ReadTunBuffSize = 2048
+	ReadTunBuffNum  = 16
+)
+
+var AndroidReadTun chan []byte // TODO: multi channel
+var AndroidWriteTun chan []byte
+
+func (t *optun) Start(localAddr string, detail *SDWANInfo) error {
+
+	return nil
+}
+
+func (t *optun) Read(bufs [][]byte, sizes []int, offset int) (n int, err error) {
+	bufs[0] = <-AndroidReadTun
+	sizes[0] = len(bufs[0])
+	return 1, nil
+}
+
+func (t *optun) Write(bufs [][]byte, offset int) (int, error) {
+	AndroidWriteTun <- bufs[0]
+	return len(bufs[0]), nil
+}
+
+func AndroidRead(data []byte, len int) {
+	head := PacketHeader{}
+	parseHeader(data, &head)
+	// gLog.dev("AndroidRead tun dst ip=%s,len=%d", net.IP{byte(head.dst >> 24), byte(head.dst >> 16), byte(head.dst >> 8), byte(head.dst)}.String(), len)
+	buf := make([]byte, len)
+	copy(buf, data)
+	AndroidReadTun <- buf
+}
+
+func AndroidWrite(buf []byte, timeoutMs int) int {
+	timeout := time.Duration(timeoutMs) * time.Millisecond
+	select {
+	case p := <-AndroidWriteTun:
+		if len(p) > int(gConf.sdwan.Mtu) {
+			gLog.e("AndroidWrite packet too large %d", len(p))
+		}
+		copy(buf, p)
+		return len(p)
+	case <-time.After(timeout):
+		return 0
+	}
+}
+
+func GetAndroidSDWANConfig(buf []byte) int {
+	p := <-AndroidSDWANConfig
+	copy(buf, p)
+	gLog.i("AndroidSDWANConfig=%s", p)
+	return len(p)
+}
+
+func GetAndroidNodeName() string {
+	gLog.i("GetAndroidNodeName=%s", gConf.Network.Node)
+	return gConf.Network.Node
+}
+
+func setTunAddr(ifname, localAddr, remoteAddr string, wintun interface{}) error {
+	// TODO:
+	return nil
+}
+
+func addRoute(dst, gw, ifname string) error {
+	// TODO:
+	return nil
+}
+
+func delRoute(dst, gw string) error {
+	// TODO:
+	return nil
+}
+
+func delRoutesByGateway(gateway string) error {
+	// TODO:
+	return nil
+}
+
+func init() {
+	AndroidReadTun = make(chan []byte, 1000)
+	AndroidWriteTun = make(chan []byte, 1000)
+}

core/optun_darwin.go

@@ -0,0 +1,90 @@
+package openp2p
+
+import (
+	"fmt"
+	"net"
+	"os/exec"
+	"strings"
+
+	"github.com/openp2p-cn/wireguard-go/tun"
+)
+
+const (
+	tunIfaceName    = "utun"
+	PIHeaderSize    = 4 // utun has no IFF_NO_PI
+	ReadTunBuffSize = 2048
+	ReadTunBuffNum  = 16
+)
+
+func (t *optun) Start(localAddr string, detail *SDWANInfo) error {
+	var err error
+	t.tunName = tunIfaceName
+	t.dev, err = tun.CreateTUN(t.tunName, int(detail.Mtu))
+	if err != nil {
+		return err
+	}
+	t.tunName, _ = t.dev.Name()
+	return nil
+}
+
+func (t *optun) Read(bufs [][]byte, sizes []int, offset int) (n int, err error) {
+	return t.dev.Read(bufs, sizes, offset)
+}
+
+func (t *optun) Write(bufs [][]byte, offset int) (int, error) {
+	return t.dev.Write(bufs, offset)
+}
+
+func setTunAddr(ifname, localAddr, remoteAddr string, wintun interface{}) error {
+	li, _, err := net.ParseCIDR(localAddr)
+	if err != nil {
+		return fmt.Errorf("parse local addr fail:%s", err)
+	}
+	ri, _, err := net.ParseCIDR(remoteAddr)
+	if err != nil {
+		return fmt.Errorf("parse remote addr fail:%s", err)
+	}
+	err = exec.Command("ifconfig", ifname, "inet", li.String(), ri.String(), "up").Run()
+	return err
+}
+
+func addRoute(dst, gw, ifname string) error {
+	err := exec.Command("route", "add", dst, gw).Run()
+	return err
+}
+
+func delRoute(dst, gw string) error {
+	err := exec.Command("route", "delete", dst, "-gateway", gw).Run()
+	return err
+}
+func delRoutesByGateway(gateway string) error {
+	cmd := exec.Command("netstat", "-rn")
+	output, err := cmd.Output()
+	if err != nil {
+		return err
+	}
+
+	lines := strings.Split(string(output), "\n")
+	for _, line := range lines {
+		if !strings.Contains(line, gateway) {
+			continue
+		}
+		fields := strings.Fields(line)
+		if len(fields) >= 2 {
+			cmd := exec.Command("route", "delete", fields[0], gateway)
+			err := cmd.Run()
+			if err != nil {
+				gLog.e("Delete route %s error:%s", fields[0], err)
+				continue
+			}
+			gLog.i("Delete route ok: %s %s\n", fields[0], gateway)
+		}
+	}
+	return nil
+}
+func addTunAddr(localAddr, remoteAddr string) error {
+	return nil
+}
+func delTunAddr(localAddr, remoteAddr string) error {
+	return nil
+}

core/optun_linux.go

@@ -0,0 +1,138 @@
+//go:build !android
+// +build !android
+
+// optun_linux.go
+package openp2p
+
+import (
+	"fmt"
+	"net"
+	"os"
+
+	"github.com/openp2p-cn/wireguard-go/tun"
+	"github.com/vishvananda/netlink"
+)
+
+const (
+	tunIfaceName = "optun"
+	PIHeaderSize = 0
+	// sdwan
+	ReadTunBuffSize = 2048
+	ReadTunBuffNum  = 16
+)
+
+var previousIP = ""
+
+func (t *optun) Start(localAddr string, detail *SDWANInfo) error {
+	var err error
+	t.tunName = tunIfaceName
+	t.dev, err = tun.CreateTUN(t.tunName, int(detail.Mtu))
+	if err != nil {
+		return err
+	}
+	err = os.WriteFile("/proc/sys/net/ipv4/ip_forward", []byte("1"), 0644)
+	if err != nil {
+		gLog.e("write ip_forward error:%s", err)
+	}
+	return nil
+}
+
+func (t *optun) Read(bufs [][]byte, sizes []int, offset int) (n int, err error) {
+	return t.dev.Read(bufs, sizes, offset)
+}
+
+func (t *optun) Write(bufs [][]byte, offset int) (int, error) {
+	return t.dev.Write(bufs, offset)
+}
+
+func setTunAddr(ifname, localAddr, remoteAddr string, wintun interface{}) error {
+	ifce, err := netlink.LinkByName(ifname)
+	if err != nil {
+		return err
+	}
+	netlink.LinkSetMTU(ifce, int(gConf.getSDWAN().Mtu))
+	netlink.LinkSetTxQLen(ifce, 1000)
+	netlink.LinkSetUp(ifce)
+
+	ln, err := netlink.ParseIPNet(localAddr)
+	if err != nil {
+		return err
+	}
+	ln.Mask = net.CIDRMask(32, 32)
+	rn, err := netlink.ParseIPNet(remoteAddr)
+	if err != nil {
+		return err
+	}
+	rn.Mask = net.CIDRMask(32, 32)
+
+	addr := &netlink.Addr{
+		IPNet: ln,
+		Peer:  rn,
+	}
+	if previousIP != "" {
+		lnDel, err := netlink.ParseIPNet(previousIP)
+		if err != nil {
+			return err
+		}
+		lnDel.Mask = net.CIDRMask(32, 32)
+
+		addrDel := &netlink.Addr{
+			IPNet: lnDel,
+			Peer:  rn,
+		}
+		netlink.AddrDel(ifce, addrDel)
+	}
+	previousIP = localAddr
+	return netlink.AddrAdd(ifce, addr)
+}
+
+func addRoute(dst, gw, ifname string) error {
+	_, networkid, err := net.ParseCIDR(dst)
+	if err != nil {
+		return err
+	}
+	ipGW := net.ParseIP(gw)
+	if ipGW == nil {
+		return fmt.Errorf("parse gateway %s failed", gw)
+	}
+	route := &netlink.Route{
+		Dst: networkid,
+		Gw:  ipGW,
+	}
+	return netlink.RouteAdd(route)
+}
+
+func delRoute(dst, gw string) error {
+	_, networkid, err := net.ParseCIDR(dst)
+	if err != nil {
+		return err
+	}
+	route := &netlink.Route{
+		Dst: networkid,
+	}
+	return netlink.RouteDel(route)
+}
+
+func delRoutesByGateway(gateway string) error {
+	ipGW := net.ParseIP(gateway)
+	if ipGW == nil {
+		return fmt.Errorf("invalid gateway IP: %s", gateway)
+	}
+
+	routes, err := netlink.RouteList(nil, netlink.FAMILY_V4)
+	if err != nil {
+		return fmt.Errorf("failed to list routes: %v", err)
+	}
+
+	for _, route := range routes {
+		if route.Gw != nil && route.Gw.Equal(ipGW) || (route.Dst != nil && route.Dst.IP.Equal(ipGW)) {
+			err := netlink.RouteDel(&route)
+			if err != nil {
+				gLog.e("Failed to delete route: %v, error: %v", route, err)
+				continue
+			}
+			gLog.i("Deleted route: %v", route)
+		}
+	}
+	return nil
+}

core/optun_other.go

@@ -0,0 +1,135 @@
+//go:build !linux && !windows && !darwin
+// +build !linux,!windows,!darwin
+
+package openp2p
+
+import (
+	"fmt"
+	"net"
+	"os/exec"
+	"strings"
+
+	"github.com/openp2p-cn/wireguard-go/tun"
+	"github.com/vishvananda/netlink"
+)
+
+const (
+	tunIfaceName    = "optun"
+	PIHeaderSize    = 0
+	ReadTunBuffSize = 2048
+	ReadTunBuffNum  = 16
+)
+
+var previousIP = ""
+
+func (t *optun) Start(localAddr string, detail *SDWANInfo) error {
+	var err error
+	t.tunName = tunIfaceName
+	t.dev, err = tun.CreateTUN(t.tunName, int(detail.Mtu))
+	if err != nil {
+		return err
+	}
+	return nil
+}
+
+func (t *optun) Read(bufs [][]byte, sizes []int, offset int) (n int, err error) {
+	return t.dev.Read(bufs, sizes, offset)
+}
+
+func (t *optun) Write(bufs [][]byte, offset int) (int, error) {
+	return t.dev.Write(bufs, offset)
+}
+
+func setTunAddr(ifname, localAddr, remoteAddr string, wintun interface{}) error {
+	ifce, err := netlink.LinkByName(ifname)
+	if err != nil {
+		return err
+	}
+	netlink.LinkSetMTU(ifce, int(gConf.getSDWAN().Mtu))
+	netlink.LinkSetTxQLen(ifce, 1000)
+	netlink.LinkSetUp(ifce)
+
+	ln, err := netlink.ParseIPNet(localAddr)
+	if err != nil {
+		return err
+	}
+	ln.Mask = net.CIDRMask(32, 32)
+	rn, err := netlink.ParseIPNet(remoteAddr)
+	if err != nil {
+		return err
+	}
+	rn.Mask = net.CIDRMask(32, 32)
+
+	addr := &netlink.Addr{
+		IPNet: ln,
+		Peer:  rn,
+	}
+	if previousIP != "" {
+		lnDel, err := netlink.ParseIPNet(previousIP)
+		if err != nil {
+			return err
+		}
+		lnDel.Mask = net.CIDRMask(32, 32)
+
+		addrDel := &netlink.Addr{
+			IPNet: lnDel,
+			Peer:  rn,
+		}
+		netlink.AddrDel(ifce, addrDel)
+	}
+	previousIP = localAddr
+	return netlink.AddrAdd(ifce, addr)
+}
+
+func addRoute(dst, gw, ifname string) error {
+	_, networkid, err := net.ParseCIDR(dst)
+	if err != nil {
+		return err
+	}
+	ipGW := net.ParseIP(gw)
+	if ipGW == nil {
+		return fmt.Errorf("parse gateway %s failed", gw)
+	}
+	route := &netlink.Route{
+		Dst: networkid,
+		Gw:  ipGW,
+	}
+	return netlink.RouteAdd(route)
+}
+
+func delRoute(dst, gw string) error {
+	_, networkid, err := net.ParseCIDR(dst)
+	if err != nil {
+		return err
+	}
+	route := &netlink.Route{
+		Dst: networkid,
+	}
+	return netlink.RouteDel(route)
+}
+
+func delRoutesByGateway(gateway string) error {
+	cmd := exec.Command("route", "-n")
+	output, err := cmd.Output()
+	if err != nil {
+		return err
+	}
+
+	lines := strings.Split(string(output), "\n")
+	for _, line := range lines {
+		if !strings.Contains(line, gateway) {
+			continue
+		}
+		fields := strings.Fields(line)
+		if len(fields) >= 8 && fields[1] == "0.0.0.0" && fields[7] == gateway {
+			delCmd := exec.Command("route", "del", "-net", fields[0], "gw", gateway)
+			err := delCmd.Run()
+			if err != nil {
+				gLog.e("Delete route %s error:%s", fields[0], err)
+				continue
+			}
+			gLog.i("Delete route ok: %s %s %s\n", fields[0], fields[1], gateway)
+		}
+	}
+	return nil
+}

core/optun_windows.go

@@ -0,0 +1,146 @@
+package openp2p
+
+import (
+	"fmt"
+	"net"
+	"net/netip"
+	"os"
+	"os/exec"
+	"path/filepath"
+	"runtime"
+	"strconv"
+	"strings"
+
+	"github.com/openp2p-cn/wireguard-go/tun"
+	"golang.org/x/sys/windows"
+	"golang.zx2c4.com/wireguard/windows/tunnel/winipcfg"
+)
+
+const (
+	tunIfaceName = "optun"
+	PIHeaderSize = 0
+	// sdwan
+	ReadTunBuffSize = 1024 * 64 // wintun will read date len > mtu, default 64k
+	ReadTunBuffNum  = 4
+)
+
+func (t *optun) Start(localAddr string, detail *SDWANInfo) error {
+	// check wintun.dll
+	tmpFile := filepath.Dir(os.Args[0]) + "/wintun.dll"
+	fs, err := os.Stat(tmpFile)
+	if err != nil || fs.Size() == 0 {
+		url := fmt.Sprintf("https://openp2p.cn/download/v1/latest/wintun/%s/wintun.dll", runtime.GOARCH)
+		err = downloadFile(url, "", tmpFile)
+		if err != nil {
+			os.Remove(tmpFile)
+			return err
+		}
+	}
+
+	t.tunName = tunIfaceName
+
+	uuid := &windows.GUID{
+		Data1: 0xf411e821,
+		Data2: 0xb310,
+		Data3: 0x4567,
+		Data4: [8]byte{0x80, 0x42, 0x83, 0x7e, 0xf4, 0x56, 0xce, 0x13},
+	}
+	t.dev, err = tun.CreateTUNWithRequestedGUID(t.tunName, uuid, int(detail.Mtu))
+	if err != nil { // retry
+		t.dev, err = tun.CreateTUNWithRequestedGUID(t.tunName, uuid, int(detail.Mtu))
+	}
+
+	if err != nil {
+		return err
+	}
+
+	return nil
+}
+
+func (t *optun) Read(bufs [][]byte, sizes []int, offset int) (n int, err error) {
+	return t.dev.Read(bufs, sizes, offset)
+}
+
+func (t *optun) Write(bufs [][]byte, offset int) (int, error) {
+	return t.dev.Write(bufs, offset)
+}
+
+func setTunAddr(ifname, localAddr, remoteAddr string, wintun interface{}) error {
+	nativeTunDevice := wintun.(*tun.NativeTun)
+	link := winipcfg.LUID(nativeTunDevice.LUID())
+	ip, err := netip.ParsePrefix(localAddr)
+	if err != nil {
+		gLog.e("ParsePrefix error:%s, luid:%d,localAddr:%s", err, nativeTunDevice.LUID(), localAddr)
+		return err
+	}
+	err = link.SetIPAddresses([]netip.Prefix{ip})
+	if err != nil {
+		gLog.e("SetIPAddresses error:%s, netip.Prefix:%+v", err, []netip.Prefix{ip})
+		return err
+	}
+	return nil
+}
+
+func addRoute(dst, gw, ifname string) error {
+	_, dstNet, err := net.ParseCIDR(dst)
+	if err != nil {
+		return err
+	}
+	i, err := net.InterfaceByName(ifname)
+	if err != nil {
+		return err
+	}
+	params := make([]string, 0)
+	params = append(params, "add")
+	params = append(params, dstNet.IP.String())
+	params = append(params, "mask")
+	params = append(params, net.IP(dstNet.Mask).String())
+	params = append(params, gw)
+	params = append(params, "if")
+	params = append(params, strconv.Itoa(i.Index))
+	// gLogger.Println(LevelINFO, "windows add route params:", params)
+	execCommand("route", true, params...)
+	return nil
+}
+
+func delRoute(dst, gw string) error {
+	_, dstNet, err := net.ParseCIDR(dst)
+	if err != nil {
+		return err
+	}
+	params := make([]string, 0)
+	params = append(params, "delete")
+	params = append(params, dstNet.IP.String())
+	params = append(params, "mask")
+	params = append(params, net.IP(dstNet.Mask).String())
+	params = append(params, gw)
+	// gLogger.Println(LevelINFO, "windows delete route params:", params)
+	execCommand("route", true, params...)
+	return nil
+}
+
+func delRoutesByGateway(gateway string) error {
+	cmd := exec.Command("route", "print", "-4")
+	output, err := cmd.Output()
+	if err != nil {
+		return err
+	}
+
+	lines := strings.Split(string(output), "\n")
+	for _, line := range lines {
+		if !strings.Contains(line, gateway) {
+			continue
+		}
+		fields := strings.Fields(line)
+		if len(fields) >= 5 {
+			cmd := exec.Command("route", "delete", fields[0], "mask", fields[1], gateway)
+			err := cmd.Run()
+			if err != nil {
+				gLog.e("Delete route %s error:%s", fields[0], err)
+				continue
+			}
+			gLog.i("Delete route ok: %s %s %s\n", fields[0], fields[1], gateway)
+		}
+	}
+	return nil
+}

core/overlay.go

@@ -5,6 +5,7 @@ import (
 	"encoding/binary"
 	"errors"
 	"net"
+	"sync"
 	"time"
 )
 
@@ -21,61 +22,57 @@ func (e *DeadlineExceededError) Error() string   { return "i/o timeout" }
 func (e *DeadlineExceededError) Timeout() bool   { return true }
 func (e *DeadlineExceededError) Temporary() bool { return true }
 
+var overlayConns sync.Map // both TCP and UDP
+func closeOverlayConns(appID uint64) {
+	overlayConns.Range(func(_, i interface{}) bool {
+		oConn := i.(*overlayConn)
+		if oConn.app.id == appID {
+			oConn.Close()
+		}
+		return true
+	})
+}
+
 // implement io.Writer
 type overlayConn struct {
-	tunnel      *P2PTunnel
+	app      *p2pApp
-	connTCP     net.Conn
+	connTCP  net.Conn
-	id          uint64
+	id       uint64
-	rtid        uint64
+	running  bool
-	running     bool
+	isClient bool
-	isClient    bool
-	appID       uint64
-	appKey      uint64
-	appKeyBytes []byte
 	// for udp
 	connUDP       *net.UDPConn
 	remoteAddr    net.Addr
-	udpRelayData  chan []byte
+	udpData       chan []byte
 	lastReadUDPTs time.Time
 }
 
 func (oConn *overlayConn) run() {
-	gLog.Printf(LvDEBUG, "%d overlayConn run start", oConn.id)
+	gLog.d("oid:%d overlayConn run start", oConn.id)
-	defer gLog.Printf(LvDEBUG, "%d overlayConn run end", oConn.id)
+	defer gLog.d("oid:%d overlayConn run end", oConn.id)
-	oConn.running = true
 	oConn.lastReadUDPTs = time.Now()
-	buffer := make([]byte, ReadBuffLen+PaddingSize)
+	buffer := make([]byte, ReadBuffLen+PaddingSize) // 16 bytes for padding
-	readBuf := buffer[:ReadBuffLen]
+	reuseBuff := buffer[:ReadBuffLen]
 	encryptData := make([]byte, ReadBuffLen+PaddingSize) // 16 bytes for padding
-	tunnelHead := new(bytes.Buffer)
+	overlayHead := new(bytes.Buffer)
-	relayHead := new(bytes.Buffer)
+
-	binary.Write(relayHead, binary.LittleEndian, oConn.rtid)
+	binary.Write(overlayHead, binary.LittleEndian, oConn.id)
-	binary.Write(tunnelHead, binary.LittleEndian, oConn.id)
+	for oConn.running && oConn.app.running {
-	for oConn.running && oConn.tunnel.isRuning() {
+		readBuff, dataLen, err := oConn.Read(reuseBuff)
-		buff, dataLen, err := oConn.Read(readBuf)
 		if err != nil {
 			if ne, ok := err.(net.Error); ok && ne.Timeout() {
 				continue
 			}
 			// overlay tcp connection normal close, debug log
-			gLog.Printf(LvDEBUG, "overlayConn %d read error:%s,close it", oConn.id, err)
+			gLog.d("oid:%d overlayConn read error:%s,close it", oConn.id, err)
 			break
 		}
-		payload := buff[:dataLen]
+		payload := readBuff[:dataLen]
-		if oConn.appKey != 0 {
+		if oConn.app.key != 0 {
-			payload, _ = encryptBytes(oConn.appKeyBytes, encryptData, buffer[:dataLen], dataLen)
+			payload, _ = encryptBytes(oConn.app.appKeyBytes, encryptData, readBuff[:dataLen], dataLen)
-		}
-		writeBytes := append(tunnelHead.Bytes(), payload...)
-		if oConn.rtid == 0 {
-			oConn.tunnel.conn.WriteBytes(MsgP2P, MsgOverlayData, writeBytes)
-			gLog.Printf(LvDEBUG, "write overlay data to %d:%d bodylen=%d", oConn.rtid, oConn.id, len(writeBytes))
-		} else {
-			// write raley data
-			all := append(relayHead.Bytes(), encodeHeader(MsgP2P, MsgOverlayData, uint32(len(writeBytes)))...)
-			all = append(all, writeBytes...)
-			oConn.tunnel.conn.WriteBytes(MsgP2P, MsgRelayData, all)
-			gLog.Printf(LvDEBUG, "write relay data to %d:%d bodylen=%d", oConn.rtid, oConn.id, len(writeBytes))
 		}
+		writeBytes := append(overlayHead.Bytes(), payload...)
+		oConn.app.WriteBytes(writeBytes)
 	}
 	if oConn.connTCP != nil {
 		oConn.connTCP.Close()
@@ -83,22 +80,17 @@ func (oConn *overlayConn) run() {
 	if oConn.connUDP != nil {
 		oConn.connUDP.Close()
 	}
-	oConn.tunnel.overlayConns.Delete(oConn.id)
+	overlayConns.Delete(oConn.id)
 	// notify peer disconnect
-	if oConn.isClient {
+	req := OverlayDisconnectReq{ID: oConn.id}
-		req := OverlayDisconnectReq{ID: oConn.id}
+	oConn.app.WriteMessage(MsgP2P, MsgOverlayDisconnectReq, &req)
-		if oConn.rtid == 0 {
-			oConn.tunnel.conn.WriteMessage(MsgP2P, MsgOverlayDisconnectReq, &req)
-		} else {
-			// write relay data
-			msg, _ := newMessage(MsgP2P, MsgOverlayDisconnectReq, &req)
-			msgWithHead := append(relayHead.Bytes(), msg...)
-			oConn.tunnel.conn.WriteBytes(MsgP2P, MsgRelayData, msgWithHead)
-		}
-	}
 }
 
-func (oConn *overlayConn) Read(reuseBuff []byte) (buff []byte, n int, err error) {
+func (oConn *overlayConn) Read(reuseBuff []byte) (buff []byte, dataLen int, err error) {
+	if !oConn.running {
+		err = ErrOverlayConnDisconnect
+		return
+	}
 	if oConn.connUDP != nil {
 		if time.Now().After(oConn.lastReadUDPTs.Add(time.Minute * 5)) {
 			err = errors.New("udp close")
@@ -106,15 +98,15 @@ func (oConn *overlayConn) Read(reuseBuff []byte) (buff []byte, n int, err error)
 		}
 		if oConn.remoteAddr != nil { // as server
 			select {
-			case buff = <-oConn.udpRelayData:
+			case buff = <-oConn.udpData:
-				n = len(buff)
+				dataLen = len(buff) - PaddingSize
 				oConn.lastReadUDPTs = time.Now()
 			case <-time.After(time.Second * 10):
 				err = ErrDeadlineExceeded
 			}
 		} else { // as client
-			oConn.connUDP.SetReadDeadline(time.Now().Add(5 * time.Second))
+			oConn.connUDP.SetReadDeadline(time.Now().Add(UDPReadTimeout))
-			n, _, err = oConn.connUDP.ReadFrom(reuseBuff)
+			dataLen, _, err = oConn.connUDP.ReadFrom(reuseBuff)
 			if err == nil {
 				oConn.lastReadUDPTs = time.Now()
 			}
@@ -122,15 +114,21 @@ func (oConn *overlayConn) Read(reuseBuff []byte) (buff []byte, n int, err error)
 		}
 		return
 	}
-	oConn.connTCP.SetReadDeadline(time.Now().Add(time.Second * 5))
+	if oConn.connTCP != nil {
-	n, err = oConn.connTCP.Read(reuseBuff)
+		oConn.connTCP.SetReadDeadline(time.Now().Add(UDPReadTimeout))
-	buff = reuseBuff
+		dataLen, err = oConn.connTCP.Read(reuseBuff)
+		buff = reuseBuff
+	}
+
 	return
 }
 
 // calling by p2pTunnel
 func (oConn *overlayConn) Write(buff []byte) (n int, err error) {
 	// add mutex when multi-thread calling
+	if !oConn.running {
+		return 0, ErrOverlayConnDisconnect
+	}
 	if oConn.connUDP != nil {
 		if oConn.remoteAddr == nil {
 			n, err = oConn.connUDP.Write(buff)
@@ -142,9 +140,26 @@ func (oConn *overlayConn) Write(buff []byte) (n int, err error) {
 		}
 		return
 	}
-	n, err = oConn.connTCP.Write(buff)
+	if oConn.connTCP != nil {
+		err = writeFull(oConn.connTCP, buff)
+		n = len(buff)
+	}
+
 	if err != nil {
 		oConn.running = false
 	}
 	return
 }
+
+func (oConn *overlayConn) Close() (err error) {
+	oConn.running = false
+	if oConn.connTCP != nil {
+		oConn.connTCP.Close()
+		// oConn.connTCP = nil
+	}
+	if oConn.connUDP != nil {
+		oConn.connUDP.Close()
+		// oConn.connUDP = nil
+	}
+	return nil
+}

core/p2papp.go

@@ -4,101 +4,568 @@ import (
 	"bytes"
 	"encoding/binary"
 	"fmt"
+	"math"
 	"math/rand"
 	"net"
 	"strconv"
 	"strings"
 	"sync"
+	"sync/atomic"
 	"time"
 )
 
+const DefaultRtt int32 = 1000
+const MaxWindowSize = 1024 * 128               // max 32k packets in flight
+const MergeAckDelay = 40                       // 40ms linux kernel tcp
+const RetransmissonTime = MergeAckDelay + 2000 // ms
+
+type appMsgCtx struct {
+	head *openP2PHeader
+	body []byte
+	ts   time.Time
+}
+
 type p2pApp struct {
 	config      AppConfig
 	listener    net.Listener
 	listenerUDP *net.UDPConn
-	tunnel      *P2PTunnel
+
-	rtid        uint64 // relay tunnelID
+	tunnelMtx sync.Mutex
-	relayNode   string
+	iptree    *IPTree // for whitelist
-	relayMode   string
+
-	hbTime      time.Time
+	hbMtx         sync.Mutex
-	hbMtx       sync.Mutex
+	running       bool
-	running     bool
+	id            uint64
-	id          uint64
+	key           uint64 // aes
-	key         uint64
+	appKeyBytes   []byte // pre-calc
-	wg          sync.WaitGroup
+	wg            sync.WaitGroup
+	msgChan       chan appMsgCtx
+	once          sync.Once
+	tunnelNum     int
+	relayIdxStart int
+	allTunnels    []*P2PTunnel
+	retryNum      []int
+	retryTime     []time.Time
+	nextRetryTime []time.Time
+	rtt           []atomic.Int32
+	relayHead     []*bytes.Buffer
+	rtid          []uint64 // peer relay tunnelID
+	relayNode     []string
+	relayMode     []string // public/private
+	hbTime        []time.Time
+	whbTime       []time.Time     // calc each tunnel rtt by hb
+	unAckSeqStart []atomic.Uint64 // record unack packet for retransmission
+	unAckSeqEnd   []atomic.Uint64
+
+	errMsg      string
+	connectTime time.Time
+	// asyncWriteChan chan []byte
+	maxWindowSize uint64
+
+	unAckTs     []atomic.Int64
+	writeTs     []atomic.Int64
+	readCacheTs atomic.Int64
+
+	seqW         uint64
+	seqR         uint64
+	seqRMtx      sync.Mutex
+	handleAckMtx sync.Mutex
+	mergeAckSeq  []atomic.Uint64
+	mergeAckTs   []atomic.Int64
+
+	preDirectSuccessIP string
 }
 
-func (app *p2pApp) isActive() bool {
+
-	if app.tunnel == nil {
+func (app *p2pApp) Tunnel(idx int) *P2PTunnel {
+	if idx > app.tunnelNum-1 {
+		return nil
+	}
+	app.tunnelMtx.Lock()
+	defer app.tunnelMtx.Unlock()
+	return app.allTunnels[idx]
+}
+
+func (app *p2pApp) SetTunnel(t *P2PTunnel, idx int) {
+	app.tunnelMtx.Lock()
+	defer app.tunnelMtx.Unlock()
+	app.allTunnels[idx] = t
+
+	app.rtt[idx].Store(DefaultRtt)
+	app.unAckTs[idx].Store(0)
+	app.writeTs[idx].Store(0)
+}
+
+func (app *p2pApp) ConnectTime() time.Time {
+	if app.allTunnels[0] != nil {
+		return app.config.connectTime
+	}
+	return app.connectTime
+}
+
+func (app *p2pApp) RetryTime() time.Time {
+	if app.allTunnels[0] != nil {
+		return app.config.retryTime
+	}
+	return app.retryTime[app.relayIdxStart]
+}
+
+func (app *p2pApp) Init(tunnelNum int) {
+	app.tunnelNum = tunnelNum
+	app.allTunnels = make([]*P2PTunnel, tunnelNum)
+	app.retryNum = make([]int, tunnelNum)
+	app.retryTime = make([]time.Time, tunnelNum)
+	app.nextRetryTime = make([]time.Time, tunnelNum)
+	app.rtt = make([]atomic.Int32, tunnelNum)
+	app.relayHead = make([]*bytes.Buffer, tunnelNum)
+	app.rtid = make([]uint64, tunnelNum)
+	app.relayNode = make([]string, tunnelNum)
+	app.relayMode = make([]string, tunnelNum)
+	app.hbTime = make([]time.Time, tunnelNum)
+	app.whbTime = make([]time.Time, tunnelNum)
+	app.unAckSeqEnd = make([]atomic.Uint64, tunnelNum)
+	app.unAckTs = make([]atomic.Int64, tunnelNum)
+	app.writeTs = make([]atomic.Int64, tunnelNum)
+	app.unAckSeqStart = make([]atomic.Uint64, tunnelNum)
+	app.mergeAckSeq = make([]atomic.Uint64, tunnelNum)
+	app.mergeAckTs = make([]atomic.Int64, tunnelNum)
+
+	app.msgChan = make(chan appMsgCtx, 50)
+	for i := 0; i < tunnelNum; i++ {
+		app.hbTime[i] = time.Now()
+	}
+	app.relayIdxStart = app.tunnelNum - 2
+	if app.relayIdxStart == 0 {
+		app.relayIdxStart = 1 // at least one direct tunnel
+	}
+	// app.unAckSeqStart.Store(0)
+	// app.mergeAckTs.Store(0)
+	// for i := 0; i < relayNum; i++ {
+	// 	app.mergeAckTsRelay[i].Store(0)
+	// }
+}
+
+func (app *p2pApp) Start(isClient bool) {
+	app.maxWindowSize = MaxWindowSize
+
+	app.PreCalcKeyBytes()
+	if isClient {
+		go app.daemonP2PTunnel()
+	}
+
+}
+
+func (app *p2pApp) daemonP2PTunnel() error {
+	for app.running {
+
+		for i := 0; i < app.relayIdxStart; i++ {
+			app.daemonDirectTunnel(i)
+			time.Sleep(time.Second)
+		}
+		for i := app.relayIdxStart; i < app.tunnelNum; i++ {
+			if i > app.relayIdxStart {
+				app.nextRetryTime[i] = time.Now().Add(time.Second * 180) // the second relay tunnel wait 3 mins
+			}
+			app.daemonRelayTunnel(i)
+			time.Sleep(time.Second)
+		}
+		time.Sleep(time.Second * 3)
+	}
+	return nil
+}
+
+func (app *p2pApp) daemonDirectTunnel(idx int) error {
+	if !GNetwork.online {
+		return nil
+	}
+	if app.config.ForceRelay == 1 && app.config.RelayNode != app.config.PeerNode {
+		return nil
+	}
+	// TODO: multi direct tunnel support symmetric NAT traversal later
+	if idx > 0 && gConf.Network.hasIPv4 == 0 && gConf.Network.hasUPNPorNATPMP == 0 && app.config.hasIPv4 == 0 && app.config.hasUPNPorNATPMP == 0 && (gConf.Network.natType == NATSymmetric || app.config.peerNatType == NATSymmetric) {
+		return nil
+	}
+	if app.Tunnel(idx) != nil && app.Tunnel(idx).isActive() {
+		return nil
+	}
+	if app.config.nextRetryTime.After(time.Now()) || app.config.Enabled == 0 {
+		return nil
+	}
+	if time.Now().Add(-time.Minute * 15).After(app.config.retryTime) { // run normally 15min, reset retrynum
+		app.retryNum[idx] = 1
+	}
+	if app.retryNum[idx] > 0 { // first time not show reconnect log
+		gLog.i("appid:%d checkDirectTunnel detect peer %s disconnect, reconnecting the %d times...", app.id, app.config.LogPeerNode(), app.retryNum[idx])
+	}
+	app.retryNum[idx]++
+	app.config.retryTime = time.Now()
+
+	app.config.connectTime = time.Now()
+	err := app.buildDirectTunnel(idx)
+	if err != nil {
+		app.config.errMsg = err.Error()
+		if err == ErrPeerOffline && app.retryNum[idx] > 2 { // stop retry, waiting for online
+			app.retryNum[idx] = retryLimit
+			gLog.i("appid:%d checkDirectTunnel %s offline, it will auto reconnect when peer node online", app.id, app.config.LogPeerNode())
+		}
+		if err == ErrBuildTunnelBusy {
+			app.retryNum[idx]--
+		}
+	}
+	interval := calcRetryTimeRelay(float64(app.retryNum[idx]))
+	if app.preDirectSuccessIP == app.config.peerIP {
+		interval = math.Min(interval, 1800) // if peerIP has been direct link succeed, retry 30min max
+	}
+	app.config.nextRetryTime = time.Now().Add(time.Duration(interval) * time.Second)
+	if app.Tunnel(idx) != nil {
+		app.preDirectSuccessIP = app.config.peerIP
+		app.once.Do(func() {
+			go app.listen()
+			// memapp also need
+			for i := app.relayIdxStart; i < app.tunnelNum; i++ {
+				go app.relayHeartbeatLoop(i)
+			}
+
+		})
+	}
+	return nil
+}
+func (app *p2pApp) buildDirectTunnel(idx int) error {
+	relayNode := ""
+	peerNatType := NATUnknown
+	peerIP := ""
+	errMsg := ""
+	var t *P2PTunnel
+	var err error
+	pn := GNetwork
+	// TODO: optimize requestPeerInfo call frequency
+	initErr := pn.requestPeerInfo(&app.config)
+	if initErr != nil {
+		gLog.w("appid:%d buildDirectTunnel %s requestPeerInfo error:%s", app.id, app.config.LogPeerNode(), initErr)
+		return initErr
+	}
+	t, err = pn.addDirectTunnel(app.config, 0, app.Tunnel(idx^1))
+	if t != nil {
+		peerNatType = t.config.peerNatType
+		peerIP = t.config.peerIP
+	}
+	if err != nil {
+		errMsg = err.Error()
+	}
+	req := ReportConnect{
+		Error:          errMsg,
+		Protocol:       app.config.Protocol,
+		SrcPort:        app.config.SrcPort,
+		NatType:        gConf.Network.natType,
+		PeerNode:       app.config.PeerNode,
+		DstPort:        app.config.DstPort,
+		DstHost:        app.config.DstHost,
+		PeerNatType:    peerNatType,
+		PeerIP:         peerIP,
+		ShareBandwidth: gConf.Network.ShareBandwidth,
+		RelayNode:      relayNode,
+		Version:        OpenP2PVersion,
+	}
+	pn.write(MsgReport, MsgReportConnect, &req)
+	if err != nil {
+		return err
+	}
+	// if rtid != 0 || t.conn.Protocol() == "tcp" {
+	// sync appkey
+	if t == nil {
+		return err
+	}
+	syncKeyReq := APPKeySync{
+		AppID:  app.id,
+		AppKey: app.key,
+	}
+	gLog.d("appid:%d buildDirectTunnel sync appkey to %s", app.id, app.config.LogPeerNode())
+	pn.push(app.config.PeerNode, MsgPushAPPKey, &syncKeyReq)
+	app.SetTunnel(t, idx)
+
+	// if memapp notify peer addmemapp
+	// if app.config.SrcPort == 0 {
+	req2 := ServerSideSaveMemApp{From: gConf.Network.Node, Node: gConf.Network.Node, TunnelID: t.id, RelayTunnelID: 0, RelayIndex: uint32(idx), TunnelNum: uint32(app.tunnelNum), AppID: app.id, AppKey: app.key, SrcPort: uint32(app.config.SrcPort)}
+	pn.push(app.config.PeerNode, MsgPushServerSideSaveMemApp, &req2)
+	gLog.d("appid:%d buildDirectTunnel push %s ServerSideSaveMemApp: %s", app.id, app.config.LogPeerNode(), prettyJson(req2))
+
+	// }
+	gLog.d("appid:%d buildDirectTunnel ok. %s use tid %d", app.id, app.config.AppName, t.id)
+	return nil
+}
+
+func (app *p2pApp) daemonRelayTunnel(idx int) error {
+	if !GNetwork.online {
+		return nil
+	}
+
+	if app.Tunnel(0) != nil && app.relayIdxStart >= 2 { // multi direct tunnel no relay
+		return nil
+	}
+	// if app.config.ForceRelay == 1 && (gConf.sdwan.CentralNode == app.config.PeerNode && compareVersion(app.config.peerVersion, SupportDualTunnelVersion) < 0) {
+	if app.config.SrcPort == 0 && (gConf.sdwan.CentralNode == app.config.PeerNode || gConf.sdwan.CentralNode == gConf.Network.Node) { // memapp central node not build relay tunnel
+		return nil
+	}
+	if gConf.sdwan.CentralNode != "" && idx != app.relayIdxStart { // if central node exist only need one relayTunnel
+		return nil
+	}
+	app.hbMtx.Lock()
+	if app.Tunnel(idx) != nil && time.Now().Before(app.hbTime[idx].Add(TunnelHeartbeatTime*2)) { // must check app.hbtime instead of relayTunnel
+		app.hbMtx.Unlock()
+		return nil
+	}
+	app.hbMtx.Unlock()
+	if app.nextRetryTime[idx].After(time.Now()) || app.config.Enabled == 0 {
+		return nil
+	}
+	if time.Now().Add(-time.Minute * 15).After(app.retryTime[idx]) { // run normally 15min, reset retrynum
+		app.retryNum[idx] = 1
+	}
+	if app.retryNum[idx] > 0 { // first time not show reconnect log
+		gLog.i("appid:%d checkRelayTunnel detect peer %s relay disconnect, reconnecting the %d times...", app.id, app.config.LogPeerNode(), app.retryNum[idx])
+	}
+	app.SetTunnel(nil, idx) // reset relayTunnel
+	app.retryNum[idx]++
+	app.retryTime[idx] = time.Now()
+	app.connectTime = time.Now()
+	err := app.buildRelayTunnel(idx)
+	if err != nil {
+		app.errMsg = err.Error()
+		if err == ErrPeerOffline && app.retryNum[idx] > 2 { // stop retry, waiting for online
+			app.retryNum[idx] = retryLimit
+			gLog.i("appid:%d checkRelayTunnel %s offline, it will auto reconnect when peer node online", app.id, app.config.LogPeerNode())
+		}
+	}
+	interval := calcRetryTimeRelay(float64(app.retryNum[idx]))
+	app.nextRetryTime[idx] = time.Now().Add(time.Duration(interval) * time.Second)
+	if app.Tunnel(idx) != nil {
+		app.once.Do(func() {
+			go app.listen()
+			// memapp also need
+			for i := 1; i < app.tunnelNum; i++ {
+				go app.relayHeartbeatLoop(i)
+			}
+		})
+	}
+	return nil
+}
+
+func (app *p2pApp) buildRelayTunnel(idx int) error {
+	var rtid uint64
+	relayNode := ""
+	relayMode := ""
+	peerNatType := NATUnknown
+	peerIP := ""
+	errMsg := ""
+	var t *P2PTunnel
+	var err error
+	pn := GNetwork
+	config := app.config
+	initErr := pn.requestPeerInfo(&config)
+	if initErr != nil {
+		gLog.w("appid:%d buildRelayTunnel %s init error:%s", app.id, config.LogPeerNode(), initErr)
+		return initErr
+	}
+	ExcludeNodes := ""
+	theOtherTunnelIdx := app.relayIdxStart
+	if idx == app.relayIdxStart {
+		theOtherTunnelIdx = app.relayIdxStart + 1
+	}
+	if app.tunnelNum > 2 && app.allTunnels[theOtherTunnelIdx] != nil {
+		ExcludeNodes = app.allTunnels[theOtherTunnelIdx].config.PeerNode
+	}
+	t, rtid, relayMode, err = pn.addRelayTunnel(config, ExcludeNodes)
+	if t != nil {
+		relayNode = t.config.PeerNode
+	}
+
+	if err != nil {
+		errMsg = err.Error()
+	}
+	if app.Tunnel(0) == nil {
+		req := ReportConnect{
+			Error:          errMsg,
+			Protocol:       config.Protocol,
+			SrcPort:        config.SrcPort,
+			NatType:        gConf.Network.natType,
+			PeerNode:       config.PeerNode,
+			DstPort:        config.DstPort,
+			DstHost:        config.DstHost,
+			PeerNatType:    peerNatType,
+			PeerIP:         peerIP,
+			ShareBandwidth: gConf.Network.ShareBandwidth,
+			RelayNode:      relayNode,
+			Version:        OpenP2PVersion,
+		}
+		pn.write(MsgReport, MsgReportConnect, &req)
+	}
+	if err != nil || t == nil {
+		return err
+	}
+
+	// if rtid != 0 || t.conn.Protocol() == "tcp" {
+	// sync appkey
+	syncKeyReq := APPKeySync{
+		AppID:  app.id,
+		AppKey: app.key,
+	}
+	gLog.d("appid:%d buildRelayTunnel sync appkey relay to %s", app.id, config.LogPeerNode())
+	pn.push(config.PeerNode, MsgPushAPPKey, &syncKeyReq)
+	app.SetRelayTunnelID(rtid, idx)
+	app.SetTunnel(t, idx)
+	app.relayNode[idx] = relayNode
+	app.relayMode[idx] = relayMode
+	app.hbTime[idx] = time.Now()
+
+	// if memapp notify peer addmemapp
+	// if config.SrcPort == 0 {
+	req2 := ServerSideSaveMemApp{From: gConf.Network.Node, Node: relayNode, TunnelID: rtid, RelayTunnelID: t.id, AppID: app.id, AppKey: app.key, RelayMode: relayMode, RelayIndex: uint32(idx), TunnelNum: uint32(app.tunnelNum), SrcPort: uint32(app.config.SrcPort)}
+	pn.push(config.PeerNode, MsgPushServerSideSaveMemApp, &req2)
+	gLog.d("appid:%d buildRelayTunnel push %s relay ServerSideSaveMemApp: %s", app.id, config.LogPeerNode(), prettyJson(req2))
+	// }
+	gLog.d("appid:%d buildRelayTunnel %s use tunnel %d", app.id, app.config.AppName, t.id)
+	return nil
+}
+
+func (app *p2pApp) buildOfficialTunnel() error {
+	return nil
+}
+
+// cache relayHead, refresh when rtid change
+func (app *p2pApp) RelayHead(idx int) *bytes.Buffer {
+	if app.relayHead[idx] == nil {
+		app.relayHead[idx] = new(bytes.Buffer)
+		binary.Write(app.relayHead[idx], binary.LittleEndian, app.rtid[idx])
+	}
+	return app.relayHead[idx]
+}
+
+func (app *p2pApp) SetRelayTunnelID(rtid uint64, idx int) {
+	app.rtid[idx] = rtid
+	app.relayHead[idx] = new(bytes.Buffer)
+	binary.Write(app.relayHead[idx], binary.LittleEndian, app.rtid[idx])
+}
+
+func (app *p2pApp) IsActive() bool {
+	if t, _ := app.AvailableTunnel(); t == nil {
+		// gLog.d("isActive app.tunnel==nil")
 		return false
 	}
-	if app.rtid == 0 { // direct mode app heartbeat equals to tunnel heartbeat
+	if app.Tunnel(0) != nil { // direct mode app heartbeat equals to tunnel heartbeat
-		return app.tunnel.isActive()
+		return app.Tunnel(0).isActive()
 	}
 	// relay mode calc app heartbeat
 	app.hbMtx.Lock()
 	defer app.hbMtx.Unlock()
-	return time.Now().Before(app.hbTime.Add(TunnelIdleTimeout))
+	if app.Tunnel(1) != nil {
+		return time.Now().Before(app.hbTime[1].Add(TunnelHeartbeatTime * 2))
+	}
+	res := time.Now().Before(app.hbTime[2].Add(TunnelHeartbeatTime * 2))
+	// if !res {
+	// 	gLog.d("%d app isActive false. peer=%s", app.id, app.config.PeerNode)
+	// }
+	return res
 }
 
-func (app *p2pApp) updateHeartbeat() {
+// only for relay tunnel heartbeat update
+func (app *p2pApp) UpdateHeartbeat(rtid uint64) {
 	app.hbMtx.Lock()
 	defer app.hbMtx.Unlock()
-	app.hbTime = time.Now()
+	for i := app.relayIdxStart; i < app.tunnelNum; i++ {
+		if rtid == app.rtid[i] || (app.Tunnel(i) != nil && app.Tunnel(i).id == rtid) {
+			app.hbTime[i] = time.Now()
+			rtt := int32(time.Since(app.whbTime[i]) / time.Millisecond)
+			preRtt := app.rtt[i].Load()
+			if preRtt != DefaultRtt {
+				rtt = int32(float64(preRtt)*(1-ma20) + float64(rtt)*ma20)
+			}
+			app.rtt[i].Store(rtt)
+			gLog.dev("appid:%d relay heartbeat %d store rtt %d", app.id, i, rtt)
+			return
+		}
+	}
+
+}
+
+func (app *p2pApp) UpdateRelayHeartbeatTs(rtid uint64) {
+	app.hbMtx.Lock()
+	defer app.hbMtx.Unlock()
+	for i := app.relayIdxStart; i < app.tunnelNum; i++ {
+		if rtid == app.rtid[i] || (app.Tunnel(i) != nil && app.Tunnel(i).id == rtid) {
+			app.whbTime[i] = time.Now()
+			return
+		}
+	}
+	// relayIdx := 1
+	// if app.tunnelNum > 2 && rtid == app.rtid[2] || (app.Tunnel(2) != nil && app.Tunnel(2).id == rtid) { // ack return rtid!=
+	// 	relayIdx = 2
+	// }
+	// app.whbTime[relayIdx] = time.Now() // one side did not write relay hb, so write whbtime in this.
 }
 
 func (app *p2pApp) listenTCP() error {
-	gLog.Printf(LvDEBUG, "tcp accept on port %d start", app.config.SrcPort)
+	gLog.d("appid:%d tcp accept on port %d start", app.id, app.config.SrcPort)
-	defer gLog.Printf(LvDEBUG, "tcp accept on port %d end", app.config.SrcPort)
+	defer gLog.d("appid:%d tcp accept on port %d end", app.id, app.config.SrcPort)
 	var err error
-	app.listener, err = net.Listen("tcp4", fmt.Sprintf("0.0.0.0:%d", app.config.SrcPort))
+	listenAddr := ""
+	if IsLocalhost(app.config.Whitelist) { // not expose port
+		listenAddr = "127.0.0.1"
+	}
+	app.listener, err = net.Listen("tcp", fmt.Sprintf("%s:%d", listenAddr, app.config.SrcPort))
 	if err != nil {
-		gLog.Printf(LvERROR, "listen error:%s", err)
+		gLog.e("appid:%d listen tcp error:%s", app.id, err)
 		return err
 	}
+	defer app.listener.Close()
 	for app.running {
 		conn, err := app.listener.Accept()
 		if err != nil {
 			if app.running {
-				gLog.Printf(LvERROR, "%d accept error:%s", app.id, err)
+				gLog.e("appid:%d  accept error:%s", app.id, err)
 			}
 			break
 		}
+		t, tidx := app.AvailableTunnel()
+		if t == nil {
+			gLog.d("appid:%d srcPort=%d, app.Tunnel()==nil, not ready", app.id, app.config.SrcPort)
+			time.Sleep(time.Second)
+			continue
+		}
+		// check white list
+		if app.config.Whitelist != "" {
+			remoteIP := conn.RemoteAddr().(*net.TCPAddr).IP.String()
+			if !app.iptree.Contains(remoteIP) && !IsLocalhost(remoteIP) {
+				conn.Close()
+				gLog.e("appid:%d %s not in whitelist, access denied", app.id, remoteIP)
+				continue
+			}
+		}
 		oConn := overlayConn{
-			tunnel:   app.tunnel,
+			app:      app,
 			connTCP:  conn,
 			id:       rand.Uint64(),
 			isClient: true,
-			rtid:     app.rtid,
+			running:  true,
-			appID:    app.id,
-			appKey:   app.key,
 		}
-		// pre-calc key bytes for encrypt
+
-		if oConn.appKey != 0 {
+		overlayConns.Store(oConn.id, &oConn)
-			encryptKey := make([]byte, AESKeySize)
+		gLog.d("appid:%d Accept TCP overlayID:%d, %s", app.id, oConn.id, oConn.connTCP.RemoteAddr())
-			binary.LittleEndian.PutUint64(encryptKey, oConn.appKey)
-			binary.LittleEndian.PutUint64(encryptKey[8:], oConn.appKey)
-			oConn.appKeyBytes = encryptKey
-		}
-		app.tunnel.overlayConns.Store(oConn.id, &oConn)
-		gLog.Printf(LvDEBUG, "Accept TCP overlayID:%d", oConn.id)
 		// tell peer connect
 		req := OverlayConnectReq{ID: oConn.id,
-			Token:    app.tunnel.pn.config.Token,
+			Token:    gConf.Network.Token,
 			DstIP:    app.config.DstHost,
 			DstPort:  app.config.DstPort,
 			Protocol: app.config.Protocol,
 			AppID:    app.id,
 		}
-		if app.rtid == 0 {
+
-			app.tunnel.conn.WriteMessage(MsgP2P, MsgOverlayConnectReq, &req)
+		if tidx != 0 {
-		} else {
+			req.RelayTunnelID = t.id
-			req.RelayTunnelID = app.tunnel.id
+		}
-			relayHead := new(bytes.Buffer)
+		app.WriteMessage(MsgP2P, MsgOverlayConnectReq, &req)
-			binary.Write(relayHead, binary.LittleEndian, app.rtid)
+		head, _ := app.ReadMessage(MsgP2P, MsgOverlayConnectRsp, time.Second*3)
-			msg, _ := newMessage(MsgP2P, MsgOverlayConnectReq, &req)
+		if head == nil {
-			msgWithHead := append(relayHead.Bytes(), msg...)
+			gLog.w("appid:%d read MsgOverlayConnectRsp error", app.id)
-			app.tunnel.conn.WriteBytes(MsgP2P, MsgRelayData, msgWithHead)
 		}
 		go oConn.run()
 	}
@@ -106,30 +573,37 @@ func (app *p2pApp) listenTCP() error {
 }
 
 func (app *p2pApp) listenUDP() error {
-	gLog.Printf(LvDEBUG, "udp accept on port %d start", app.config.SrcPort)
+	gLog.d("appid:%d udp accept on port %d start", app.id, app.config.SrcPort)
-	defer gLog.Printf(LvDEBUG, "udp accept on port %d end", app.config.SrcPort)
+	defer gLog.d("appid:%d udp accept on port %d end", app.id, app.config.SrcPort)
 	var err error
 	app.listenerUDP, err = net.ListenUDP("udp", &net.UDPAddr{IP: net.IPv4zero, Port: app.config.SrcPort})
 	if err != nil {
-		gLog.Printf(LvERROR, "listen error:%s", err)
+		gLog.e("appid:%d listen udp error:%s", app.id, err)
 		return err
 	}
-	buffer := make([]byte, 64*1024)
+	defer app.listenerUDP.Close()
+	buffer := make([]byte, 64*1024+PaddingSize)
 	udpID := make([]byte, 8)
-	for {
+	for app.running {
-		app.listenerUDP.SetReadDeadline(time.Now().Add(time.Second * 10))
+		app.listenerUDP.SetReadDeadline(time.Now().Add(UDPReadTimeout))
 		len, remoteAddr, err := app.listenerUDP.ReadFrom(buffer)
 		if err != nil {
 			if ne, ok := err.(net.Error); ok && ne.Timeout() {
 				continue
 			} else {
-				gLog.Printf(LvERROR, "udp read failed:%s", err)
+				gLog.e("appid:%d udp read failed:%s", app.id, err)
 				break
 			}
 		} else {
-			b := bytes.Buffer{}
+			t, tidx := app.AvailableTunnel()
-			b.Write(buffer[:len])
+			if t == nil {
-			// load from app.tunnel.overlayConns by remoteAddr error, new udp connection
+				gLog.d("appid:%d srcPort=%d, app.Tunnel()==nil, not ready", app.id, app.config.SrcPort)
+				time.Sleep(time.Second)
+				continue
+			}
+			dupData := bytes.Buffer{} // should uses memory pool
+			dupData.Write(buffer[:len+PaddingSize])
+			// load from app.overlayConns by remoteAddr error, new udp connection
 			remoteIP := strings.Split(remoteAddr.String(), ":")[0]
 			port, _ := strconv.Atoi(strings.Split(remoteAddr.String(), ":")[1])
 			a := net.ParseIP(remoteIP)
@@ -139,83 +613,74 @@ func (app *p2pApp) listenUDP() error {
 			udpID[3] = a[3]
 			udpID[4] = byte(port)
 			udpID[5] = byte(port >> 8)
-			id := binary.LittleEndian.Uint64(udpID)
+			id := binary.LittleEndian.Uint64(udpID) // convert remoteIP:port to uint64
-			s, ok := app.tunnel.overlayConns.Load(id)
+			s, ok := overlayConns.Load(id)
 			if !ok {
 				oConn := overlayConn{
-					tunnel:       app.tunnel,
+					app:        app,
-					connUDP:      app.listenerUDP,
+					connUDP:    app.listenerUDP,
-					remoteAddr:   remoteAddr,
+					remoteAddr: remoteAddr,
-					udpRelayData: make(chan []byte, 1000),
+					udpData:    make(chan []byte, 1000),
-					id:           id,
+					id:         id,
-					isClient:     true,
+					isClient:   true,
-					rtid:         app.rtid,
+					running:    true,
-					appID:        app.id,
-					appKey:       app.key,
 				}
-				// calc key bytes for encrypt
+				overlayConns.Store(oConn.id, &oConn)
-				if oConn.appKey != 0 {
+				gLog.d("appid:%d Accept UDP overlayID:%d", app.id, oConn.id)
-					encryptKey := make([]byte, AESKeySize)
-					binary.LittleEndian.PutUint64(encryptKey, oConn.appKey)
-					binary.LittleEndian.PutUint64(encryptKey[8:], oConn.appKey)
-					oConn.appKeyBytes = encryptKey
-				}
-				app.tunnel.overlayConns.Store(oConn.id, &oConn)
-				gLog.Printf(LvDEBUG, "Accept UDP overlayID:%d", oConn.id)
 				// tell peer connect
 				req := OverlayConnectReq{ID: oConn.id,
-					Token:    app.tunnel.pn.config.Token,
+					Token:    gConf.Network.Token,
 					DstIP:    app.config.DstHost,
 					DstPort:  app.config.DstPort,
 					Protocol: app.config.Protocol,
 					AppID:    app.id,
 				}
-				if app.rtid == 0 {
+				if tidx != 0 {
-					app.tunnel.conn.WriteMessage(MsgP2P, MsgOverlayConnectReq, &req)
+					req.RelayTunnelID = t.id
-				} else {
+				}
-					req.RelayTunnelID = app.tunnel.id
+				app.WriteMessage(MsgP2P, MsgOverlayConnectReq, &req)
-					relayHead := new(bytes.Buffer)
+				head, _ := app.ReadMessage(MsgP2P, MsgOverlayConnectRsp, time.Second*3)
-					binary.Write(relayHead, binary.LittleEndian, app.rtid)
+				if head == nil {
-					msg, _ := newMessage(MsgP2P, MsgOverlayConnectReq, &req)
+					gLog.w("appid:%d read MsgOverlayConnectRsp error", app.id)
-					msgWithHead := append(relayHead.Bytes(), msg...)
-					app.tunnel.conn.WriteBytes(MsgP2P, MsgRelayData, msgWithHead)
 				}
 				go oConn.run()
-				oConn.udpRelayData <- b.Bytes()
+				oConn.udpData <- dupData.Bytes()
 			}
 
-			// load from app.tunnel.overlayConns by remoteAddr ok, write relay data
+			// load from overlayConns by remoteAddr ok, write relay data
 			overlayConn, ok := s.(*overlayConn)
 			if !ok {
 				continue
 			}
-			overlayConn.udpRelayData <- b.Bytes()
+			overlayConn.udpData <- dupData.Bytes()
 		}
 	}
 	return nil
 }
 
 func (app *p2pApp) listen() error {
-	gLog.Printf(LvINFO, "LISTEN ON PORT %s:%d START", app.config.Protocol, app.config.SrcPort)
+	if app.config.SrcPort == 0 {
-	defer gLog.Printf(LvINFO, "LISTEN ON PORT %s:%d END", app.config.Protocol, app.config.SrcPort)
+		return nil
+	}
+	gLog.i("appid:%d LISTEN ON PORT %s:%d START", app.id, app.config.Protocol, app.config.SrcPort)
+	defer gLog.i("appid:%d LISTEN ON PORT %s:%d END", app.id, app.config.Protocol, app.config.SrcPort)
 	app.wg.Add(1)
 	defer app.wg.Done()
-	app.running = true
+	for app.running {
-	if app.rtid != 0 {
-		go app.relayHeartbeatLoop()
-	}
-	for app.tunnel.isRuning() && app.running {
 		if app.config.Protocol == "udp" {
 			app.listenUDP()
 		} else {
 			app.listenTCP()
 		}
+		if !app.running {
+			break
+		}
 		time.Sleep(time.Second * 10)
 	}
 	return nil
 }
 
-func (app *p2pApp) close() {
+func (app *p2pApp) Close() {
 	app.running = false
 	if app.listener != nil {
 		app.listener.Close()
@@ -223,26 +688,193 @@ func (app *p2pApp) close() {
 	if app.listenerUDP != nil {
 		app.listenerUDP.Close()
 	}
-	if app.tunnel != nil {
+	closeOverlayConns(app.id)
-		app.tunnel.closeOverlayConns(app.id)
-	}
 	app.wg.Wait()
 }
 
 // TODO: many relay app on the same P2PTunnel will send a lot of relay heartbeat
-func (app *p2pApp) relayHeartbeatLoop() {
+func (app *p2pApp) relayHeartbeatLoop(idx int) {
 	app.wg.Add(1)
 	defer app.wg.Done()
-	gLog.Printf(LvDEBUG, "relayHeartbeat to %d start", app.rtid)
+	gLog.d("appid:%d %s relayHeartbeat to rtid:%d start", app.id, app.config.LogPeerNode(), app.rtid[idx])
-	defer gLog.Printf(LvDEBUG, "relayHeartbeat to %d end", app.rtid)
+	defer gLog.d("appid:%d %s relayHeartbeat to rtid%d end", app.id, app.config.LogPeerNode(), app.rtid[idx])
-	relayHead := new(bytes.Buffer)
+
-	binary.Write(relayHead, binary.LittleEndian, app.rtid)
+	for app.running {
-	req := RelayHeartbeat{RelayTunnelID: app.tunnel.id,
+		if app.Tunnel(idx) == nil || !app.Tunnel(idx).isRuning() {
-		AppID: app.id}
+			time.Sleep(TunnelHeartbeatTime)
-	msg, _ := newMessage(MsgP2P, MsgRelayHeartbeat, &req)
+			continue
-	msgWithHead := append(relayHead.Bytes(), msg...)
+		}
-	for app.tunnel.isRuning() && app.running {
+		req := RelayHeartbeat{From: gConf.Network.Node, RelayTunnelID: app.Tunnel(idx).id, RelayTunnelID2: app.rtid[idx],
-		app.tunnel.conn.WriteBytes(MsgP2P, MsgRelayData, msgWithHead)
+			AppID: app.id}
+		err := app.Tunnel(idx).WriteMessage(app.rtid[idx], MsgP2P, MsgRelayHeartbeat, &req)
+		if err != nil {
+			gLog.e("appid:%d %s rtid:%d write relay tunnel heartbeat error %s", app.id, app.config.LogPeerNode(), app.rtid[idx], err)
+			app.SetTunnel(nil, idx)
+			continue
+		}
+		app.whbTime[idx] = time.Now()
+		// TODO: debug relay heartbeat
+		gLog.dev("appid:%d %s rtid:%d write relay tunnel heartbeat ok", app.id, app.config.LogPeerNode(), app.rtid[idx])
 		time.Sleep(TunnelHeartbeatTime)
 	}
 }
+
+func (app *p2pApp) WriteMessage(mainType uint16, subType uint16, req interface{}) error {
+	t, tidx := app.AvailableTunnel()
+	if t == nil {
+		return ErrAppWithoutTunnel
+	}
+	return t.WriteMessage(app.rtid[tidx], mainType, subType, req)
+}
+
+func (app *p2pApp) WriteMessageWithAppID(mainType uint16, subType uint16, req interface{}) error {
+	t, tidx := app.AvailableTunnel()
+	if t == nil {
+		return ErrAppWithoutTunnel
+	}
+	appID := app.id
+	if app.config.SrcPort == 0 {
+		appID = NodeNameToID(app.config.PeerNode)
+	}
+	return t.WriteMessageWithAppID(appID, app.rtid[tidx], mainType, subType, req)
+}
+
+func (app *p2pApp) WriteBytes(data []byte) error {
+	t, tidx := app.AvailableTunnel()
+	if t == nil {
+		return ErrAppWithoutTunnel
+	}
+	if tidx < app.relayIdxStart { // direct mode
+		return t.conn.WriteBytes(MsgP2P, MsgOverlayData, data)
+	}
+	all := append(app.relayHead[tidx].Bytes(), encodeHeader(MsgP2P, MsgOverlayData, uint32(len(data)))...)
+	all = append(all, data...)
+	t.conn.WriteBytes(MsgP2P, MsgRelayData, all)
+
+	return nil
+}
+
+func (app *p2pApp) PreCalcKeyBytes() {
+	// pre-calc key bytes for encrypt
+	if app.key != 0 {
+		encryptKey := make([]byte, AESKeySize)
+		binary.LittleEndian.PutUint64(encryptKey, app.key)
+		binary.LittleEndian.PutUint64(encryptKey[8:], app.key)
+		app.appKeyBytes = encryptKey
+	}
+}
+
+func (app *p2pApp) WriteNodeDataMP(IPPacket []byte) (err error) {
+	t, tidx := app.fastestTunnel()
+	if t == nil {
+		return ErrAppWithoutTunnel
+	}
+	dataWithSeq := new(bytes.Buffer)
+	binary.Write(dataWithSeq, binary.LittleEndian, gConf.nodeID())
+	binary.Write(dataWithSeq, binary.LittleEndian, app.seqW)
+	dataWithSeq.Write(IPPacket)
+	// gLog.d("DEBUG writeTs=%d, unAckSeqStart=%d", wu.writeTs.UnixMilli(), app.unAckSeqStart[tidx].Load())
+
+	if tidx < app.relayIdxStart { // direct mode
+		t.asyncWriteNodeData(gConf.nodeID(), app.seqW, IPPacket, nil)
+		gLog.dev("appid:%d asyncWriteDirect IPPacket len=%d", app.id, len(IPPacket))
+	} else {
+		t.asyncWriteNodeData(gConf.nodeID(), app.seqW, IPPacket, app.RelayHead(tidx).Bytes())
+		gLog.dev("appid:%d asyncWriteRelay%d IPPacket len=%d", app.id, tidx, len(IPPacket))
+	}
+	app.seqW++
+	return err
+}
+
+func (app *p2pApp) handleNodeDataMP(seq uint64, data []byte, t *P2PTunnel) {
+		GNetwork.nodeData <- data
+
+}
+
+func (app *p2pApp) isReliable() bool {
+	// return app.config.SrcPort != 0
+	return true
+}
+
+func (app *p2pApp) AvailableTunnel() (*P2PTunnel, int) {
+	for i := 0; i < app.tunnelNum; i++ {
+		if app.allTunnels[i] != nil {
+			return app.allTunnels[i], i
+		}
+	}
+	return nil, 0
+}
+
+func (app *p2pApp) fastestTunnel() (t *P2PTunnel, idx int) {
+	// gLog.d("appid:%d fastestTunnel %d %d",app.id, app.DirectRTT(), app.MinRelayRTT())
+	if gConf.Network.specTunnel > 0 {
+		if app.Tunnel(gConf.Network.specTunnel) != nil {
+			return app.Tunnel(gConf.Network.specTunnel), gConf.Network.specTunnel
+		}
+	}
+
+	for i := 0; i < app.tunnelNum; i++ {
+		if app.Tunnel(i) != nil {
+			t = app.Tunnel(i)
+			idx = i
+			break
+		}
+	}
+	return
+}
+
+func (app *p2pApp) ResetWindow() {
+	app.seqW = 0
+	app.seqR = 0
+	for i := 0; i < app.tunnelNum; i++ {
+		app.unAckSeqEnd[i].Store(0)
+		app.unAckTs[i].Store(0)
+		app.writeTs[i].Store(0)
+	}
+
+}
+
+func (app *p2pApp) Retry(all bool) {
+	gLog.d("appid:%d retry app %s", app.id, app.config.LogPeerNode())
+	for i := 0; i < app.tunnelNum; i++ {
+		app.retryNum[i] = 0
+		app.nextRetryTime[i] = time.Now()
+		if all && i == 0 {
+			app.hbMtx.Lock()
+			app.hbTime[i] = time.Now().Add(-TunnelHeartbeatTime * 3)
+			app.hbMtx.Unlock()
+			// app.config.retryNum = 0
+			app.config.nextRetryTime = time.Now()
+			app.ResetWindow()
+		}
+	}
+}
+
+func (app *p2pApp) StoreMessage(head *openP2PHeader, body []byte) {
+	app.msgChan <- appMsgCtx{head, body, time.Now()}
+}
+
+func (app *p2pApp) ReadMessage(mainType uint16, subType uint16, timeout time.Duration) (head *openP2PHeader, body []byte) {
+	for {
+		select {
+		case <-time.After(timeout):
+			gLog.e("appid:%d app.ReadMessage error %d:%d timeout", app.id, mainType, subType)
+			return
+		case msg := <-app.msgChan:
+			if time.Since(msg.ts) > ReadMsgTimeout {
+				gLog.d("appid:%d app.ReadMessage error expired %d:%d", app.id, mainType, subType)
+				continue
+			}
+			if msg.head.MainType != mainType || msg.head.SubType != subType {
+				gLog.d("appid:%d app.ReadMessage error type %d:%d, requeue it", app.id, msg.head.MainType, msg.head.SubType)
+				app.msgChan <- msg
+				time.Sleep(time.Second)
+				continue
+			}
+			head = msg.head
+			body = msg.body[8:]
+			return
+		}
+	}
+}
+

core/p2pnetwork.go

@@ -1,714 +1,1256 @@
-package openp2p
+package openp2p
-
+
-import (
+import (
-	"bytes"
+	"bytes"
-	"crypto/tls"
+	"context"
-	"encoding/binary"
+	"crypto/tls"
-	"encoding/json"
+	"crypto/x509"
-	"errors"
+	"encoding/binary"
-	"fmt"
+	"encoding/json"
-	"math"
+	"errors"
-	"math/rand"
+	"fmt"
-	"net/http"
+	"math/rand"
-	"net/url"
+	"net"
-	"strings"
+	"net/http"
-	"sync"
+	"os"
-	"time"
+	"runtime"
-
+
-	"github.com/gorilla/websocket"
+	// _ "net/http/pprof"
-)
+	"net/url"
-
+	"reflect"
-var (
+	"strings"
-	instance *P2PNetwork
+	"sync"
-	once     sync.Once
+	"time"
-)
+
-
+	"github.com/gorilla/websocket"
-type P2PNetwork struct {
+)
-	conn      *websocket.Conn
+
-	online    bool
+var (
-	running   bool
+	v4l            *v4Listener
-	restartCh chan bool
+	onceP2PNetwork sync.Once
-	wg        sync.WaitGroup
+)
-	writeMtx  sync.Mutex
+
-	serverTs  int64
+const (
-	localTs   int64
+	retryLimit                  = 20
-	hbTime    time.Time
+	retryInterval               = 10 * time.Second
-	// msgMap    sync.Map
+	DefaultLoginMaxDelaySeconds = 60
-	msgMap     map[uint64]chan []byte //key: nodeID
+	MsgQueueSize                = 256
-	msgMapMtx  sync.Mutex
+)
-	config     NetworkConfig
+
-	allTunnels sync.Map
+// golang not support float64 const
-	apps       sync.Map //key: protocol+srcport; value: p2pApp
+var (
-	limiter    *BandwidthLimiter
+	ma20 float64 = 1.0 / 20
-}
+	ma10 float64 = 1.0 / 10
-
+	ma5  float64 = 1.0 / 5
-func P2PNetworkInstance(config *NetworkConfig) *P2PNetwork {
+)
-	if instance == nil {
+
-		once.Do(func() {
+type NodeData struct {
-			instance = &P2PNetwork{
+	NodeID uint64 // unused
-				restartCh: make(chan bool, 2),
+	Data   []byte
-				online:    false,
+}
-				running:   true,
+
-				msgMap:    make(map[uint64]chan []byte),
+type P2PNetwork struct {
-				limiter:   newBandwidthLimiter(config.ShareBandwidth),
+	conn          *websocket.Conn
-			}
+	online        bool
-			instance.msgMap[0] = make(chan []byte) // for gateway
+	running       bool
-			if config != nil {
+	restartCh     chan bool
-				instance.config = *config
+	wgReconnect   sync.WaitGroup
-			}
+	writeMtx      sync.Mutex
-			instance.init()
+	reqGatewayMtx sync.Mutex
-			go instance.run()
+	hbTime        time.Time
-		})
+	initTime      time.Time
-	}
+	// for sync server time
-	return instance
+	t1     int64 // nanoSeconds
-}
+	preRtt int64 // nanoSeconds
-
+	dt     int64 // client faster then server dt nanoSeconds
-func (pn *P2PNetwork) run() {
+	ddtma  int64
-	go pn.autorunApp()
+	ddt    int64    // differential of dt
-	heartbeatTimer := time.NewTicker(NetworkHeartbeatTime)
+	msgMap sync.Map //key: nodeID
-	for pn.running {
+	// msgMap     map[uint64]chan pushMsg //key: nodeID
-		select {
+	allTunnels           sync.Map // key: tid
-		case <-heartbeatTimer.C:
+	apps                 sync.Map //key: peerid when memapp for sdwan node data indicate app/random uint64 when portforward; value: *p2pApp
-			pn.write(MsgHeartbeat, 0, "")
+	limiter              *SpeedLimiter
-
+	nodeData             chan []byte
-		case <-pn.restartCh:
+	sdwan                *p2pSDWAN
-			pn.online = false
+	tunnelCloseCh        chan *P2PTunnel
-			pn.wg.Wait() // wait read/write goroutine exited
+	loginMaxDelaySeconds int
-			time.Sleep(NetworkHeartbeatTime)
+	peerNodeMutex        sync.Map
-			err := pn.init()
+}
-			if err != nil {
+
-				gLog.Println(LvERROR, "P2PNetwork init error:", err)
+type msgCtx struct {
-			}
+	data []byte
-		}
+	ts   time.Time
-	}
+}
-}
+
-
+func P2PNetworkInstance() {
-func (pn *P2PNetwork) Connect(timeout int) bool {
+	if GNetwork == nil {
-	// waiting for login response
+		onceP2PNetwork.Do(func() {
-	for i := 0; i < (timeout / 1000); i++ {
+			GNetwork = &P2PNetwork{
-		if pn.hbTime.After(time.Now().Add(-NetworkHeartbeatTime)) {
+				restartCh:            make(chan bool, 1),
-			return true
+				tunnelCloseCh:        make(chan *P2PTunnel, 100),
-		}
+				nodeData:             make(chan []byte, 10000),
-		time.Sleep(time.Second)
+				online:               false,
-	}
+				running:              true,
-	return false
+				limiter:              newSpeedLimiter(gConf.Network.ShareBandwidth*1024*1024/8, 1),
-}
+				dt:                   0,
-
+				ddt:                  0,
-func (pn *P2PNetwork) runAll() {
+				loginMaxDelaySeconds: DefaultLoginMaxDelaySeconds,
-	gConf.mtx.Lock() // lock for copy gConf.Apps and the modification of config(it's pointer)
+				initTime:             time.Now(),
-	defer gConf.mtx.Unlock()
+			}
-	allApps := gConf.Apps // read a copy, other thread will modify the gConf.Apps
+			GNetwork.msgMap.Store(uint64(0), make(chan msgCtx, MsgQueueSize)) // for gateway
-
+			GNetwork.StartSDWAN()
-	for _, config := range allApps {
+			v4l = &v4Listener{port: gConf.Network.PublicIPPort}
-		if config.nextRetryTime.After(time.Now()) {
+			go GNetwork.keepAlive() // init() will block, keepalive should before init
-			continue
+			GNetwork.init()
-		}
+			go GNetwork.run()
-		if config.Enabled == 0 {
+
-			continue
+			go func() {
-		}
+				ticker := time.NewTicker(10 * time.Minute)
-		if config.AppName == "" {
+				defer ticker.Stop()
-			config.AppName = fmt.Sprintf("%s%d", config.Protocol, config.SrcPort)
+				for range ticker.C {
-		}
+					dumpStack()
-		appExist := false
+				}
-		i, ok := pn.apps.Load(fmt.Sprintf("%s%d", config.Protocol, config.SrcPort))
+			}()
-		if ok {
+			go func() {
-			app := i.(*p2pApp)
+				for {
-			appExist = true
+					time.Sleep(time.Hour)
-			if app.isActive() {
+					oldIPv6 := gConf.IPv6()
-				continue
+					GNetwork.refreshIPv6()
-			}
+					newIPv6 := gConf.IPv6()
-		}
+					if oldIPv6 != newIPv6 {
-		if appExist {
+						req := ReportBasic{
-			pn.DeleteApp(*config)
+							Mac:             gConf.Network.mac,
-		}
+							LanIP:           gConf.Network.localIP,
-		if config.retryNum > 0 {
+							OS:              gConf.Network.os,
-			gLog.Printf(LvINFO, "detect app %s disconnect, reconnecting the %d times...", config.AppName, config.retryNum)
+							HasIPv4:         gConf.Network.hasIPv4,
-			if time.Now().Add(-time.Minute * 15).After(config.retryTime) { // normal lasts 15min
+							HasUPNPorNATPMP: gConf.Network.hasUPNPorNATPMP,
-				config.retryNum = 0
+							Version:         OpenP2PVersion,
-			}
+							IPv6:            newIPv6,
-		}
+							PublicIPPort:    gConf.Network.PublicIPPort,
-		config.retryNum++
+						}
-		config.retryTime = time.Now()
+						GNetwork.write(MsgReport, MsgReportBasic, &req)
-		increase := math.Pow(1.5, float64(config.retryNum)) // exponential increase retry time. 1.5^x
+					}
-		if increase > 900 {
+				}
-			increase = 900
+			}()
-			config.Enabled = 0
+			cleanTempFiles()
-			gLog.Printf(LvWARN, "app %s has stopped retry, manually enable it on Web console", config.AppName)
+			// go func() {
-			continue
+			// 	log.Println("Starting pprof server on :16060")
-		}
+			// 	log.Println(http.ListenAndServe("0.0.0.0:16060", nil))
-		config.nextRetryTime = time.Now().Add(time.Second * time.Duration(increase))
+			// }()
-		config.connectTime = time.Now()
+		})
-		config.peerToken = pn.config.Token
+	}
-		gConf.mtx.Unlock() // AddApp will take a period of time
+}
-		err := pn.AddApp(*config)
+
-		gConf.mtx.Lock()
+func (pn *P2PNetwork) keepAlive() {
-		if err != nil {
+	gLog.i("P2PNetwork keepAlive start")
-			config.errMsg = err.Error()
+	for {
-		}
+		time.Sleep(time.Second * 10)
-	}
+
-}
+		if pn.hbTime.Before(time.Now().Add(-NetworkHeartbeatTime * 3)) {
-func (pn *P2PNetwork) autorunApp() {
+			if pn.initTime.After(time.Now().Add(-NetworkHeartbeatTime * 3)) {
-	gLog.Println(LvINFO, "autorunApp start")
+				gLog.d("Init less than 3 mins, skipping this check")
-	for pn.running {
+				continue
-		time.Sleep(time.Second)
+			}
-		if !pn.online {
+			gLog.e("P2PNetwork keepAlive error, exit worker")
-			continue
+			dumpStack()
-		}
+			if !isAndroid() {
-		pn.runAll()
+				os.Exit(9)
-		time.Sleep(time.Second * 10)
+			}
-	}
+		}
-	gLog.Println(LvINFO, "autorunApp end")
+	}
-}
+}
-
+
-func (pn *P2PNetwork) addRelayTunnel(config AppConfig) (*P2PTunnel, uint64, string, error) {
+func dumpStack() {
-	gLog.Printf(LvINFO, "addRelayTunnel to %s start", config.PeerNode)
+	buf := make([]byte, 1024*1024)
-	defer gLog.Printf(LvINFO, "addRelayTunnel to %s end", config.PeerNode)
+	n := runtime.Stack(buf, true)
-	// request a relay node or specify manually(TODO)
+	tmpFile := "./log/stack.log.tmp"
-	pn.write(MsgRelay, MsgRelayNodeReq, &RelayNodeReq{config.PeerNode})
+	if err := os.WriteFile(tmpFile, buf[:n], 0644); err != nil {
-	head, body := pn.read("", MsgRelay, MsgRelayNodeRsp, time.Second*10)
+		gLog.e("print runtime.Stack error")
-	if head == nil {
+		return
-		return nil, 0, "", errors.New("read MsgRelayNodeRsp error")
+	}
-	}
+	os.Rename(tmpFile, "./log/stack.log")
-	rsp := RelayNodeRsp{}
+}
-	err := json.Unmarshal(body, &rsp)
+
-	if err != nil {
+func (pn *P2PNetwork) run() {
-		gLog.Printf(LvERROR, "wrong RelayNodeRsp:%s", err)
+	heartbeatTimer := time.NewTicker(NetworkHeartbeatTime)
-		return nil, 0, "", errors.New("unmarshal MsgRelayNodeRsp error")
+	pn.t1 = time.Now().UnixNano()
-	}
+	pn.write(MsgHeartbeat, 0, "")
-	if rsp.RelayName == "" || rsp.RelayToken == 0 {
+	for {
-		gLog.Printf(LvERROR, "MsgRelayNodeReq error")
+		select {
-		return nil, 0, "", errors.New("MsgRelayNodeReq error")
+		case <-heartbeatTimer.C:
-	}
+			pn.t1 = time.Now().UnixNano()
-	gLog.Printf(LvINFO, "got relay node:%s", rsp.RelayName)
+			pn.write(MsgHeartbeat, 0, "")
-	relayConfig := config
+		case isRestartDelay := <-pn.restartCh:
-	relayConfig.PeerNode = rsp.RelayName
+			gLog.i("got restart channel")
-	relayConfig.peerToken = rsp.RelayToken
+			// pn.sdwan.reset()
-	///
+			pn.online = false
-	t, err := pn.addDirectTunnel(relayConfig, 0)
+			waitDone := make(chan struct{})
-	if err != nil {
+			go func() {
-		gLog.Println(LvERROR, "direct connect error:", err)
+				defer close(waitDone)
-		return nil, 0, "", err
+				pn.wgReconnect.Wait()
-	}
+			}()
-	// notify peer addRelayTunnel
+
-	req := AddRelayTunnelReq{
+			select {
-		From:       pn.config.Node,
+			case <-waitDone:
-		RelayName:  rsp.RelayName,
+			case <-time.After(30 * time.Second):
-		RelayToken: rsp.RelayToken,
+				gLog.e("pn.wgReconnect.Wait() timeout, mostly websocket hang. restart client")
-	}
+				os.Exit(0)
-	gLog.Printf(LvINFO, "push relay %s---------%s", config.PeerNode, rsp.RelayName)
+			}
-	pn.push(config.PeerNode, MsgPushAddRelayTunnelReq, &req)
+
-
+			if isRestartDelay {
-	// wait relay ready
+				delay := ClientAPITimeout + time.Duration(rand.Int()%pn.loginMaxDelaySeconds)*time.Second
-	head, body = pn.read(config.PeerNode, MsgPush, MsgPushAddRelayTunnelRsp, PeerAddRelayTimeount) // TODO: const value
+				time.Sleep(delay)
-	if head == nil {
+			}
-		gLog.Printf(LvERROR, "read MsgPushAddRelayTunnelRsp error")
+			err := pn.init()
-		return nil, 0, "", errors.New("read MsgPushAddRelayTunnelRsp error")
+			if err != nil {
-	}
+				gLog.e("P2PNetwork init error:%s", err)
-	rspID := TunnelMsg{}
+			}
-	err = json.Unmarshal(body, &rspID)
+			gConf.retryAllApp()
-	if err != nil {
+
-		gLog.Printf(LvERROR, "wrong RelayNodeRsp:%s", err)
+		case t := <-pn.tunnelCloseCh:
-		return nil, 0, "", errors.New("unmarshal MsgRelayNodeRsp error")
+			gLog.d("got tunnelCloseCh %s", t.config.LogPeerNode())
-	}
+			pn.apps.Range(func(id, i interface{}) bool {
-	return t, rspID.ID, rsp.Mode, err
+				app := i.(*p2pApp)
-}
+				for i := 0; i < app.tunnelNum; i++ {
-
+					if app.Tunnel(i) == t {
-// use *AppConfig to save status
+						app.SetTunnel(nil, i)
-func (pn *P2PNetwork) AddApp(config AppConfig) error {
+					}
-	gLog.Printf(LvINFO, "addApp %s to %s:%s:%d start", config.AppName, config.PeerNode, config.DstHost, config.DstPort)
+				}
-	defer gLog.Printf(LvINFO, "addApp %s to %s:%s:%d end", config.AppName, config.PeerNode, config.DstHost, config.DstPort)
+				return true
-	if !pn.online {
+			})
-		return errors.New("P2PNetwork offline")
+		}
-	}
+	}
-	// check if app already exist?
+}
-	appExist := false
+
-	_, ok := pn.apps.Load(fmt.Sprintf("%s%d", config.Protocol, config.SrcPort))
+func (pn *P2PNetwork) NotifyTunnelClose(t *P2PTunnel) bool {
-	if ok {
+	select {
-		appExist = true
+	case pn.tunnelCloseCh <- t:
-	}
+		return true
-	if appExist {
+	default:
-		return errors.New("P2PApp already exist")
+	}
-	}
+	return false
-	appID := rand.Uint64()
+}
-	appKey := uint64(0)
+
-	var rtid uint64
+func (pn *P2PNetwork) Connect(timeout int) bool {
-	relayNode := ""
+	// waiting for heartbeat
-	relayMode := ""
+	for i := 0; i < (timeout / 1000); i++ {
-	peerNatType := NATUnknown
+		if pn.hbTime.After(time.Now().Add(-NetworkHeartbeatTime)) {
-	peerIP := ""
+			return true
-	errMsg := ""
+		}
-	t, err := pn.addDirectTunnel(config, 0)
+		time.Sleep(time.Second)
-	if t != nil {
+	}
-		peerNatType = t.config.peerNatType
+	return false
-		peerIP = t.config.peerIP
+}
-	}
+
-	// TODO: if tcp failed, should try udp punching, nattype should refactor also, when NATNONE and failed we don't know the peerNatType
+func (pn *P2PNetwork) runAll() {
-
+	gConf.mtx.RLock() // lock for coRUpy gConf.Apps and the modification of config(it's pointer)
-	if err != nil && err == ErrorHandshake {
+	defer gConf.mtx.RUnlock()
-		gLog.Println(LvERROR, "direct connect failed, try to relay")
+	allApps := gConf.Apps // read a copy, other thread will modify the gConf.Apps
-		t, rtid, relayMode, err = pn.addRelayTunnel(config)
+	for _, config := range allApps {
-		if t != nil {
+		if config.Enabled == 0 {
-			relayNode = t.config.PeerNode
+			continue
-		}
+		}
-	}
+		if app := pn.findApp(config); app != nil {
-
+			// update some attribute
-	if err != nil {
+			app.config.PunchPriority = config.PunchPriority
-		errMsg = err.Error()
+			app.config.UnderlayProtocol = config.UnderlayProtocol
-	}
+			app.config.RelayNode = config.RelayNode
-	req := ReportConnect{
+			continue
-		Error:          errMsg,
+		}
-		Protocol:       config.Protocol,
+
-		SrcPort:        config.SrcPort,
+		// config.peerToken = gConf.Network.Token // move to AddApp
-		NatType:        pn.config.natType,
+		gConf.mtx.RUnlock() // AddApp will take a period of time, let outside modify gConf
-		PeerNode:       config.PeerNode,
+		pn.AddApp(*config)
-		DstPort:        config.DstPort,
+		gConf.mtx.RLock()
-		DstHost:        config.DstHost,
+
-		PeerNatType:    peerNatType,
+	}
-		PeerIP:         peerIP,
+}
-		ShareBandwidth: pn.config.ShareBandwidth,
+
-		RelayNode:      relayNode,
+func (pn *P2PNetwork) autorunApp() {
-		Version:        OpenP2PVersion,
+	gLog.i("autorunApp start")
-	}
+	pn.wgReconnect.Add(1)
-	pn.write(MsgReport, MsgReportConnect, &req)
+	defer pn.wgReconnect.Done()
-	if err != nil {
+	for pn.running && pn.online {
-		return err
+		time.Sleep(time.Second)
-	}
+		pn.runAll()
-	if rtid != 0 || t.conn.Protocol() == "tcp" {
+	}
-		// sync appkey
+	gLog.i("autorunApp end")
-		appKey = rand.Uint64()
+}
-		req := APPKeySync{
+
-			AppID:  appID,
+func (pn *P2PNetwork) addRelayTunnel(config AppConfig, excludeNodes string) (*P2PTunnel, uint64, string, error) {
-			AppKey: appKey,
+	gLog.d("addRelayTunnel to %s start", config.LogPeerNode())
-		}
+	defer gLog.d("addRelayTunnel to %s end", config.LogPeerNode())
-		gLog.Printf(LvINFO, "sync appkey to %s", config.PeerNode)
+	var relayTunnel *P2PTunnel
-		pn.push(config.PeerNode, MsgPushAPPKey, &req)
+	relayConfig := AppConfig{
-	}
+		peerToken:        config.peerToken,
-	app := p2pApp{
+		PunchPriority:    config.PunchPriority,
-		id:        appID,
+		UnderlayProtocol: config.UnderlayProtocol,
-		key:       appKey,
+		relayMode:        "private",
-		tunnel:    t,
+	}
-		config:    config,
+	if config.RelayNode != excludeNodes {
-		rtid:      rtid,
+		relayConfig.PeerNode = config.RelayNode
-		relayNode: relayNode,
+		// TODO: verify relay node is online
-		relayMode: relayMode,
+	}
-		hbTime:    time.Now()}
+	if relayConfig.PeerNode == "" {
-	pn.apps.Store(fmt.Sprintf("%s%d", config.Protocol, config.SrcPort), &app)
+		// find existing relay tunnel
-	if err == nil {
+		pn.apps.Range(func(id, i interface{}) bool {
-		go app.listen()
+			app := i.(*p2pApp)
-	}
+			if app.config.PeerNode != config.PeerNode {
-	return err
+				return true
-}
+			}
-
+			for i := 1; i < app.tunnelNum; i++ { // index 1 for relay tunnel
-func (pn *P2PNetwork) DeleteApp(config AppConfig) {
+				if app.Tunnel(i) != nil && app.Tunnel(i).config.PeerNode != excludeNodes && time.Now().Before(app.hbTime[i].Add(TunnelHeartbeatTime*2)) {
-	gLog.Printf(LvINFO, "DeleteApp %s%d start", config.Protocol, config.SrcPort)
+					relayConfig.PeerNode = app.Tunnel(i).config.PeerNode
-	defer gLog.Printf(LvINFO, "DeleteApp %s%d end", config.Protocol, config.SrcPort)
+					relayConfig.relayMode = app.Tunnel(i).config.relayMode
-	// close the apps of this config
+					relayTunnel = app.Tunnel(i)
-	i, ok := pn.apps.Load(fmt.Sprintf("%s%d", config.Protocol, config.SrcPort))
+					gLog.d("found existing relay tunnel %s", relayConfig.LogPeerNode())
-	if ok {
+					return false
-		app := i.(*p2pApp)
+				}
-		gLog.Printf(LvINFO, "app %s exist, delete it", fmt.Sprintf("%s%d", config.Protocol, config.SrcPort))
+			}
-		app.close()
+			return true
-		pn.apps.Delete(fmt.Sprintf("%s%d", config.Protocol, config.SrcPort))
+		})
-	}
+		if relayConfig.PeerNode == "" { // request relay node
-}
+			pn.reqGatewayMtx.Lock()
-
+			pn.write(MsgRelay, MsgRelayNodeReq, &RelayNodeReq{config.PeerNode, excludeNodes})
-func (pn *P2PNetwork) addDirectTunnel(config AppConfig, tid uint64) (*P2PTunnel, error) {
+			head, body := pn.read("", MsgRelay, MsgRelayNodeRsp, ClientAPITimeout)
-	gLog.Printf(LvDEBUG, "addDirectTunnel %s%d to %s:%s:%d start", config.Protocol, config.SrcPort, config.PeerNode, config.DstHost, config.DstPort)
+			pn.reqGatewayMtx.Unlock()
-	defer gLog.Printf(LvDEBUG, "addDirectTunnel %s%d to %s:%s:%d end", config.Protocol, config.SrcPort, config.PeerNode, config.DstHost, config.DstPort)
+			if head == nil {
-	isClient := false
+				return nil, 0, "", errors.New("read MsgRelayNodeRsp error")
-	// client side tid=0, assign random uint64
+			}
-	if tid == 0 {
+			rsp := RelayNodeRsp{}
-		tid = rand.Uint64()
+			if err := json.Unmarshal(body, &rsp); err != nil {
-		isClient = true
+				return nil, 0, "", errors.New("unmarshal MsgRelayNodeRsp error")
-	}
+			}
-	exist := false
+			if rsp.RelayName == "" || rsp.RelayToken == 0 {
-	// find existing tunnel to peer
+				gLog.e("MsgRelayNodeReq error")
-	var t *P2PTunnel
+				return nil, 0, "", errors.New("MsgRelayNodeReq error")
-	pn.allTunnels.Range(func(id, i interface{}) bool {
+			}
-		t = i.(*P2PTunnel)
+			relayConfig.PeerNode = rsp.RelayName
-		if t.config.PeerNode == config.PeerNode {
+			relayConfig.peerToken = rsp.RelayToken
-			// server side force close existing tunnel
+			relayConfig.relayMode = rsp.Mode
-			if !isClient {
+			gLog.d("got relay node:%s", relayConfig.LogPeerNode())
-				t.close()
+		}
-				return false
+
-			}
+	}
-
+	///
-			// client side checking
+	if relayTunnel == nil {
-			gLog.Println(LvINFO, "tunnel already exist ", config.PeerNode)
+		var err error
-			isActive := t.checkActive()
+		relayTunnel, err = pn.addDirectTunnel(relayConfig, 0, nil)
-			// inactive, close it
+		if err != nil || relayTunnel == nil {
-			if !isActive {
+			gLog.w("direct connect error:%s", err)
-				gLog.Println(LvINFO, "but it's not active, close it ", config.PeerNode)
+			if err != nil && config.RelayNode != "" {
-				t.close()
+				return nil, 0, "", err // let outside known the specified relay node offline, than stop retry
-			} else {
+			}
-				// active
+			return nil, 0, "", ErrConnectRelayNode // relay offline will stop retry
-				exist = true
+		}
-			}
+	}
-			return false
+
-		}
+	// notify peer addRelayTunnel
-		return true
+	req := AddRelayTunnelReq{
-	})
+		From:             gConf.Network.Node,
-	if exist {
+		RelayName:        relayConfig.PeerNode,
-		return t, nil
+		RelayToken:       relayConfig.peerToken,
-	}
+		RelayMode:        relayConfig.relayMode,
-	// create tunnel if not exist
+		RelayTunnelID:    relayTunnel.id,
-	t = &P2PTunnel{pn: pn,
+		PunchPriority:    relayConfig.PunchPriority,
-		config: config,
+		UnderlayProtocol: relayConfig.UnderlayProtocol,
-		id:     tid,
+	}
-	}
+
-	pn.msgMapMtx.Lock()
+	gLog.d("push %s the relay node(%s)", config.LogPeerNode(), relayConfig.LogPeerNode())
-	pn.msgMap[nodeNameToID(config.PeerNode)] = make(chan []byte, 50)
+	pn.push(config.PeerNode, MsgPushAddRelayTunnelReq, &req)
-	pn.msgMapMtx.Unlock()
+
-	// server side
+	// wait relay ready
-	if !isClient {
+	head, body := pn.read(config.PeerNode, MsgPush, MsgPushAddRelayTunnelRsp, PeerAddRelayTimeount)
-		err := pn.newTunnel(t, tid, isClient)
+	if head == nil {
-		return t, err // always return
+		gLog.e("read MsgPushAddRelayTunnelRsp error")
-	}
+		return nil, 0, "", errors.New("read MsgPushAddRelayTunnelRsp error")
-	// client side
+	}
-	// peer info
+	rspID := TunnelMsg{}
-	initErr := t.requestPeerInfo()
+	if err := json.Unmarshal(body, &rspID); err != nil {
-	if initErr != nil {
+		gLog.d("Unmarshal error:%s", ErrPeerConnectRelay)
-		gLog.Println(LvERROR, "init error:", initErr)
+		return nil, 0, "", ErrPeerConnectRelay
-		return nil, initErr
+	}
-	}
+	return relayTunnel, rspID.ID, relayConfig.relayMode, nil
-	err := ErrorHandshake
+}
-	// try TCP6
+
-	if IsIPv6(t.config.peerIPv6) && IsIPv6(t.pn.config.publicIPv6) {
+// use *AppConfig to save status
-		gLog.Println(LvINFO, "try TCP6")
+func (pn *P2PNetwork) AddApp(config AppConfig) error {
-		t.config.linkMode = LinkModeTCP6
+	config.peerToken = gConf.Network.Token
-		t.config.isUnderlayServer = 0
+	gLog.i("addApp %s to %s:%s:%d start", config.AppName, config.LogPeerNode(), config.DstHost, config.DstPort)
-		if err = pn.newTunnel(t, tid, isClient); err == nil {
+	defer gLog.i("addApp %s to %s:%s:%d end", config.AppName, config.LogPeerNode(), config.DstHost, config.DstPort)
-			return t, nil
+	if !pn.online {
-		}
+		return errors.New("P2PNetwork offline")
-	}
+	}
-
+	if _, ok := pn.msgMap.Load(NodeNameToID(config.PeerNode)); !ok {
-	// TODO: try UDP6
+		pn.msgMap.Store(NodeNameToID(config.PeerNode), make(chan msgCtx, MsgQueueSize))
-
+	}
-	// try TCP4
+	// check if app already exist?
-	if t.config.hasIPv4 == 1 || t.pn.config.hasIPv4 == 1 || t.config.hasUPNPorNATPMP == 1 || t.pn.config.hasUPNPorNATPMP == 1 {
+	existApp := pn.findApp(&config)
-		gLog.Println(LvINFO, "try TCP4")
+	if existApp != nil {
-		t.config.linkMode = LinkModeTCP4
+		if existApp.tunnelNum == int(gConf.sdwan.TunnelNum) {
-		if t.config.hasIPv4 == 1 || t.config.hasUPNPorNATPMP == 1 {
+			return errors.New("P2PApp already exist")
-			t.config.isUnderlayServer = 0
+		} else {
-		} else {
+			gLog.d("app %s exist but tunnelNum changed from %d to %d, delete it and recreate", existApp.config.AppName, existApp.tunnelNum, gConf.sdwan.TunnelNum)
-			t.config.isUnderlayServer = 1
+			pn.DeleteApp(config)
-		}
+		}
-		if err = pn.newTunnel(t, tid, isClient); err == nil {
+
-			return t, nil
+	}
-		}
+
-	}
+	app := p2pApp{
-	// TODO: try UDP4
+		// tunnel:    t,
-
+		id:      rand.Uint64(),
-	// try TCPPunch
+		key:     rand.Uint64(),
-	if t.config.peerNatType == NATCone && t.pn.config.natType == NATCone { // TODO: support c2s
+		config:  config,
-		gLog.Println(LvINFO, "try TCP4 Punch")
+		iptree:  NewIPTree(config.Whitelist),
-		t.config.linkMode = LinkModeTCPPunch
+		running: true,
-		t.config.isUnderlayServer = 0
+		// asyncWriteChan: make(chan []byte, WriteDataChanSize),
-		if err = pn.newTunnel(t, tid, isClient); err == nil {
+	}
-			return t, nil
+	if config.SrcPort == 0 {
-		}
+		app.id = NodeNameToID(config.PeerNode)
-	}
+	}
-	// try UDPPunch
+	tunnelNum := 2
-	if t.config.peerNatType == NATCone || t.pn.config.natType == NATCone {
+
-		gLog.Println(LvINFO, "try UDP4 Punch")
+	app.Init(tunnelNum)
-		t.config.linkMode = LinkModeUDPPunch
+	if _, ok := pn.msgMap.Load(NodeNameToID(config.PeerNode)); !ok {
-		t.config.isUnderlayServer = 0
+		pn.msgMap.Store(NodeNameToID(config.PeerNode), make(chan msgCtx, MsgQueueSize))
-		if err = pn.newTunnel(t, tid, isClient); err == nil {
+	}
-			return t, nil
+	app.Start(true)
-		}
+	pn.apps.Store(app.id, &app) // TODO: store appid
-	}
+	gLog.d("Store app %d", app.id)
-	return nil, err
+
-}
+	return nil
-
+}
-func (pn *P2PNetwork) newTunnel(t *P2PTunnel, tid uint64, isClient bool) error {
+
-	t.initPort()
+func (pn *P2PNetwork) findApp(config *AppConfig) (app *p2pApp) {
-	if isClient {
+	pn.apps.Range(func(id, i interface{}) bool {
-		if err := t.connect(); err != nil {
+		tempApp := i.(*p2pApp)
-			gLog.Println(LvERROR, "p2pTunnel connect error:", err)
+		if config.SrcPort == 0 { // sdwan app
-			return err
+			if tempApp.config.SrcPort == config.SrcPort &&
-		}
+				tempApp.config.PeerNode == config.PeerNode {
-	} else {
+				app = tempApp
-		if err := t.listen(); err != nil {
+				return false
-			gLog.Println(LvERROR, "p2pTunnel listen error:", err)
+			}
-			return err
+		} else { // portforward app
-		}
+			if tempApp.config.SrcPort == config.SrcPort &&
-	}
+				tempApp.config.Protocol == config.Protocol {
-	// store it when success
+				app = tempApp
-	gLog.Printf(LvDEBUG, "store tunnel %d", tid)
+				return false
-	pn.allTunnels.Store(tid, t)
+			}
-	return nil
+		}
-}
+		return true
-func (pn *P2PNetwork) init() error {
+	})
-	gLog.Println(LvINFO, "init start")
+	return
-	var err error
+}
-	for {
+
-		// detect nat type
+func (pn *P2PNetwork) DeleteApp(config AppConfig) {
-		pn.config.publicIP, pn.config.natType, pn.config.hasIPv4, pn.config.hasUPNPorNATPMP, err = getNATType(pn.config.ServerHost, pn.config.UDPPort1, pn.config.UDPPort2)
+	gLog.i("DeleteApp %s to %s:%s:%d start", config.AppName, config.LogPeerNode(), config.DstHost, config.DstPort)
-		// for testcase
+	defer gLog.i("DeleteApp %s to %s:%s:%d end", config.AppName, config.LogPeerNode(), config.DstHost, config.DstPort)
-		if strings.Contains(pn.config.Node, "openp2pS2STest") {
+	// close the apps of this config
-			pn.config.natType = NATSymmetric
+	if tempApp := pn.findApp(&config); tempApp != nil {
-			pn.config.hasIPv4 = 0
+		gLog.i("app %s exist, delete it", tempApp.config.AppName)
-			pn.config.hasUPNPorNATPMP = 0
+		tempApp.Close()
-
+		if config.SrcPort != 0 {
-		}
+			pn.apps.Delete(tempApp.id)
-		if strings.Contains(pn.config.Node, "openp2pC2CTest") {
+		} else {
-			pn.config.natType = NATCone
+			pn.apps.Delete(NodeNameToID(config.PeerNode))
-			pn.config.hasIPv4 = 0
+		}
-			pn.config.hasUPNPorNATPMP = 0
+
-		}
+	}
-		if err != nil {
+
-			gLog.Println(LvDEBUG, "detect NAT type error:", err)
+}
-			break
+
-		}
+func (pn *P2PNetwork) findTunnel(peerNode string, ignoredTunnel *P2PTunnel) (t *P2PTunnel) {
-		gLog.Println(LvDEBUG, "detect NAT type:", pn.config.natType, " publicIP:", pn.config.publicIP)
+	t = nil
-		gatewayURL := fmt.Sprintf("%s:%d", pn.config.ServerHost, pn.config.ServerPort)
+	// find existing tunnel to peer
-		uri := "/openp2p/v1/login"
+	pn.allTunnels.Range(func(id, i interface{}) bool {
-		config := tls.Config{InsecureSkipVerify: true} // let's encrypt root cert "DST Root CA X3" expired at 2021/09/29. many old system(windows server 2008 etc) will not trust our cert
+		tmpt := i.(*P2PTunnel)
-		websocket.DefaultDialer.TLSClientConfig = &config
+		if tmpt.config.PeerNode == peerNode && tmpt != ignoredTunnel {
-		u := url.URL{Scheme: "wss", Host: gatewayURL, Path: uri}
+			gLog.d("tunnel already exist %s", tmpt.config.LogPeerNode())
-		q := u.Query()
+			isActive := tmpt.checkActive()
-		q.Add("node", pn.config.Node)
+			// inactive, close it
-		q.Add("token", fmt.Sprintf("%d", pn.config.Token))
+			if !isActive {
-		q.Add("version", OpenP2PVersion)
+				gLog.i("but it's not active, close it %s", tmpt.config.LogPeerNode())
-		q.Add("nattype", fmt.Sprintf("%d", pn.config.natType))
+				tmpt.close()
-		q.Add("sharebandwidth", fmt.Sprintf("%d", pn.config.ShareBandwidth))
+			} else {
-		u.RawQuery = q.Encode()
+				t = tmpt
-		var ws *websocket.Conn
+			}
-		ws, _, err = websocket.DefaultDialer.Dial(u.String(), nil)
+			return false
-		if err != nil {
+		}
-			break
+		return true
-		}
+	})
-		pn.online = true
+	return t
-		pn.conn = ws
+}
-		localAddr := strings.Split(ws.LocalAddr().String(), ":")
+
-		if len(localAddr) == 2 {
+func (pn *P2PNetwork) addDirectTunnel(config AppConfig, tid uint64, ignoredTunnel *P2PTunnel) (t *P2PTunnel, err error) {
-			pn.config.localIP = localAddr[0]
+	gLog.d("addDirectTunnel %s%d to %s:%s:%d tid:%d start", config.Protocol, config.SrcPort, config.LogPeerNode(), config.DstHost, config.DstPort, tid)
-		} else {
+	defer gLog.d("addDirectTunnel %s%d to %s:%s:%d tid:%d end", config.Protocol, config.SrcPort, config.LogPeerNode(), config.DstHost, config.DstPort, tid)
-			err = errors.New("get local ip failed")
+
-			break
+	nodeID := NodeNameToID(config.PeerNode)
-		}
+	mutex, _ := pn.peerNodeMutex.LoadOrStore(nodeID, &sync.Mutex{})
-		go pn.readLoop()
+	mutex.(*sync.Mutex).Lock()
-		pn.config.mac = getmac(pn.config.localIP)
+	defer mutex.(*sync.Mutex).Unlock()
-		pn.config.os = getOsName()
+
-
+	isClient := false
-		req := ReportBasic{
+	// client side tid=0, assign random uint64
-			Mac:             pn.config.mac,
+	if tid == 0 {
-			LanIP:           pn.config.localIP,
+		tid = rand.Uint64()
-			OS:              pn.config.os,
+		isClient = true
-			HasIPv4:         pn.config.hasIPv4,
+	}
-			HasUPNPorNATPMP: pn.config.hasUPNPorNATPMP,
+
-			Version:         OpenP2PVersion,
+	if _, ok := pn.msgMap.Load(nodeID); !ok {
-		}
+		pn.msgMap.Store(nodeID, make(chan msgCtx, MsgQueueSize))
-		rsp := netInfo()
+	}
-		gLog.Println(LvDEBUG, "netinfo:", rsp)
+
-		if rsp != nil && rsp.Country != "" {
+	if isClient { // only client side find existing tunnel, server side should force build tunnel
-			if IsIPv6(rsp.IP.String()) {
+		if existTunnel := pn.findTunnel(config.PeerNode, ignoredTunnel); existTunnel != nil {
-				pn.config.publicIPv6 = rsp.IP.String()
+			return existTunnel, nil
-			}
+		}
-			req.NetInfo = *rsp
+	}
-		} else {
+
-			pn.refreshIPv6(true)
+	// server side
-		}
+	if !isClient {
-		req.IPv6 = pn.config.publicIPv6
+		t, err = pn.newTunnel(config, tid, isClient, ignoredTunnel)
-		pn.write(MsgReport, MsgReportBasic, &req)
+		return t, err // always return
-		gLog.Println(LvDEBUG, "P2PNetwork init ok")
+	}
-		break
+
-	}
+	// client side
-	if err != nil {
+	// peer info
-		// init failed, retry
+	initErr := pn.requestPeerInfo(&config)
-		pn.restartCh <- true
+	if initErr != nil {
-		gLog.Println(LvERROR, "P2PNetwork init error:", err)
+		gLog.w("%s init error:%s", config.LogPeerNode(), initErr)
-	}
+		return nil, initErr
-	return err
+	}
-}
+
-
+	gLog.d("config.peerNode=%s,config.peerVersion=%s,config.peerIP=%s,config.peerLanIP=%s,gConf.Network.publicIP=%s,config.peerIPv6=%s,config.hasIPv4=%d,config.hasUPNPorNATPMP=%d,gConf.Network.hasIPv4=%d,gConf.Network.hasUPNPorNATPMP=%d,config.peerNatType=%d,gConf.Network.natType=%d,config.PunchPriority=%d,IPv6=%s",
-func (pn *P2PNetwork) handleMessage(t int, msg []byte) {
+		config.LogPeerNode(), config.peerVersion, config.peerIP, config.peerLanIP, gConf.Network.publicIP, config.peerIPv6, config.hasIPv4, config.hasUPNPorNATPMP, gConf.Network.hasIPv4, gConf.Network.hasUPNPorNATPMP, config.peerNatType, gConf.Network.natType, config.PunchPriority, gConf.IPv6())
-	head := openP2PHeader{}
+
-	err := binary.Read(bytes.NewReader(msg[:openP2PHeaderSize]), binary.LittleEndian, &head)
+	// try Intranet
-	if err != nil {
+	if config.peerIP == gConf.Network.publicIP && compareVersion(config.peerVersion, SupportIntranetVersion) >= 0 { // old version client has no peerLanIP
-		gLog.Println(LvERROR, "handleMessage error:", err)
+		gLog.i("try Intranet")
-		return
+		config.linkMode = LinkModeIntranet
-	}
+		config.isUnderlayServer = 0
-	switch head.MainType {
+		if t, err = pn.newTunnel(config, tid, isClient, ignoredTunnel); err == nil {
-	case MsgLogin:
+			return t, nil
-		// gLog.Println(LevelINFO,string(msg))
+		}
-		rsp := LoginRsp{}
+	}
-		err = json.Unmarshal(msg[openP2PHeaderSize:], &rsp)
+	thisTunnelForcev6 := false
-		if err != nil {
+	// try TCP6
-			gLog.Printf(LvERROR, "wrong login response:%s", err)
+	if !strings.Contains(gConf.Network.Node, "openp2pS2STest") && IsIPv6(config.peerIPv6) && IsIPv6(gConf.IPv6()) && (config.PunchPriority&PunchPriorityUDPOnly == 0) {
-			return
+		gLog.i("try TCP6")
-		}
+		config.linkMode = LinkModeTCP6
-		if rsp.Error != 0 {
+		config.isUnderlayServer = 0
-			gLog.Printf(LvERROR, "login error:%d, detail:%s", rsp.Error, rsp.Detail)
+		if gConf.Forcev6 {
-			pn.running = false
+			thisTunnelForcev6 = true
-		} else {
+		}
-			pn.serverTs = rsp.Ts
+		if t, err = pn.newTunnel(config, tid, isClient, ignoredTunnel); err == nil {
-			pn.hbTime = time.Now()
+			return t, nil
-			pn.config.Token = rsp.Token
+		}
-			pn.config.User = rsp.User
+	}
-			gConf.setToken(rsp.Token)
+
-			gConf.setUser(rsp.User)
+	// try UDP6? maybe no
-			if len(rsp.Node) >= MinNodeNameLen {
+
-				gConf.setNode(rsp.Node)
+	// try IPv4
-			}
+	if !thisTunnelForcev6 && !strings.Contains(gConf.Network.Node, "openp2pS2STest") && (config.hasIPv4 == 1 || gConf.Network.hasIPv4 == 1 || config.hasUPNPorNATPMP == 1 || gConf.Network.hasUPNPorNATPMP == 1) {
-			gConf.save()
+		if config.PunchPriority&PunchPriorityUDPOnly != 0 && compareVersion(config.peerVersion, SupportUDP4DirectVersion) >= 0 {
-			pn.localTs = time.Now().Unix()
+			gLog.i("try UDP4")
-			gLog.Printf(LvINFO, "login ok. user=%s,node=%s,Server ts=%d, local ts=%d", rsp.User, rsp.Node, rsp.Ts, pn.localTs)
+			config.linkMode = LinkModeUDP4
-		}
+		} else {
-	case MsgHeartbeat:
+			gLog.i("try TCP4")
-		gLog.Printf(LvDEBUG, "P2PNetwork heartbeat ok")
+			config.linkMode = LinkModeTCP4
-		pn.hbTime = time.Now()
+		}
-	case MsgPush:
+
-		handlePush(pn, head.SubType, msg)
+		if gConf.Network.hasIPv4 == 1 || gConf.Network.hasUPNPorNATPMP == 1 {
-	default:
+			config.isUnderlayServer = 1
-		pn.msgMapMtx.Lock()
+		} else {
-		ch := pn.msgMap[0]
+			config.isUnderlayServer = 0
-		pn.msgMapMtx.Unlock()
+		}
-		ch <- msg
+		if t, err = pn.newTunnel(config, tid, isClient, ignoredTunnel); err == nil {
-		return
+			return t, nil
-	}
+		}
-}
+	}
-
+	// try UDP4? maybe no
-func (pn *P2PNetwork) readLoop() {
+	var primaryPunchFunc func() (*P2PTunnel, error)
-	gLog.Printf(LvDEBUG, "P2PNetwork readLoop start")
+	var secondaryPunchFunc func() (*P2PTunnel, error)
-	pn.wg.Add(1)
+	funcUDP := func() (t *P2PTunnel, err error) {
-	defer pn.wg.Done()
+		if thisTunnelForcev6 || config.PunchPriority&PunchPriorityTCPOnly != 0 {
-	for pn.running {
+			return
-		pn.conn.SetReadDeadline(time.Now().Add(NetworkHeartbeatTime + 10*time.Second))
+		}
-		t, msg, err := pn.conn.ReadMessage()
+		// try UDPPunch
-		if err != nil {
+		for i := 0; i < Cone2ConeUDPPunchMaxRetry; i++ { // when both 2 nats has restrict firewall, simultaneous punching needs to be very precise, it takes a few tries
-			gLog.Printf(LvERROR, "P2PNetwork read error:%s", err)
+			if config.peerNatType == NATCone || gConf.Network.natType == NATCone {
-			pn.conn.Close()
+				gLog.i("try UDP4 Punch")
-			pn.restartCh <- true
+				config.linkMode = LinkModeUDPPunch
-			break
+				config.isUnderlayServer = 0
-		}
+				if t, err = pn.newTunnel(config, tid, isClient, ignoredTunnel); err == nil {
-		pn.handleMessage(t, msg)
+					return t, nil
-	}
+				}
-	gLog.Printf(LvDEBUG, "P2PNetwork readLoop end")
+			}
-}
+			if !(config.peerNatType == NATCone && gConf.Network.natType == NATCone) { // not cone2cone, no more try
-
+				break
-func (pn *P2PNetwork) write(mainType uint16, subType uint16, packet interface{}) error {
+			}
-	if !pn.online {
+		}
-		return errors.New("P2P network offline")
+		return
-	}
+	}
-	msg, err := newMessage(mainType, subType, packet)
+	funcTCP := func() (t *P2PTunnel, err error) {
-	if err != nil {
+		if thisTunnelForcev6 || config.PunchPriority&PunchPriorityUDPOnly != 0 {
-		return err
+			return
-	}
+		}
-	pn.writeMtx.Lock()
+		// try TCPPunch
-	defer pn.writeMtx.Unlock()
+		for i := 0; i < Cone2ConeTCPPunchMaxRetry; i++ { // when both 2 nats has restrict firewall, simultaneous punching needs to be very precise, it takes a few tries
-	if err = pn.conn.WriteMessage(websocket.BinaryMessage, msg); err != nil {
+			if config.peerNatType == NATCone || gConf.Network.natType == NATCone {
-		gLog.Printf(LvERROR, "write msgType %d,%d error:%s", mainType, subType, err)
+				gLog.i("try TCP4 Punch")
-		pn.conn.Close()
+				config.linkMode = LinkModeTCPPunch
-	}
+				config.isUnderlayServer = 0
-	return err
+				if t, err = pn.newTunnel(config, tid, isClient, ignoredTunnel); err == nil {
-}
+					gLog.i("TCP4 Punch ok")
-
+					return t, nil
-func (pn *P2PNetwork) relay(to uint64, body []byte) error {
+				}
-	gLog.Printf(LvDEBUG, "relay data to %d", to)
+			}
-	i, ok := pn.allTunnels.Load(to)
+		}
-	if !ok {
+		return
-		return nil
+	}
-	}
+	if config.PunchPriority&PunchPriorityTCPFirst != 0 {
-	tunnel := i.(*P2PTunnel)
+		primaryPunchFunc = funcTCP
-	if tunnel.config.shareBandwidth > 0 {
+		secondaryPunchFunc = funcUDP
-		pn.limiter.Add(len(body))
+	} else {
-	}
+		primaryPunchFunc = funcUDP
-	tunnel.conn.WriteBuffer(body)
+		secondaryPunchFunc = funcTCP
-	return nil
+	}
-}
+	if t, err = primaryPunchFunc(); t != nil && err == nil {
-
+		return t, err
-func (pn *P2PNetwork) push(to string, subType uint16, packet interface{}) error {
+	}
-	gLog.Printf(LvDEBUG, "push msgType %d to %s", subType, to)
+	if t, err = secondaryPunchFunc(); t != nil && err == nil {
-	if !pn.online {
+		return t, err
-		return errors.New("client offline")
+	}
-	}
+
-	pushHead := PushHeader{}
+	// TODO: s2s won't return err
-	pushHead.From = nodeNameToID(pn.config.Node)
+	return nil, err
-	pushHead.To = nodeNameToID(to)
+}
-	pushHeadBuf := new(bytes.Buffer)
+
-	err := binary.Write(pushHeadBuf, binary.LittleEndian, pushHead)
+func (pn *P2PNetwork) newTunnel(config AppConfig, tid uint64, isClient bool, ignoredTunnel *P2PTunnel) (t *P2PTunnel, err error) {
-	if err != nil {
+	if isClient { // only client side find existing tunnel, server side should force build tunnel
-		return err
+		if existTunnel := pn.findTunnel(config.PeerNode, ignoredTunnel); existTunnel != nil {
-	}
+			return existTunnel, nil
-	data, err := json.Marshal(packet)
+		}
-	if err != nil {
+	}
-		return err
+
-	}
+	t = &P2PTunnel{
-	// gLog.Println(LevelINFO,"write packet:", string(data))
+		config:         config,
-	pushMsg := append(encodeHeader(MsgPush, subType, uint32(len(data)+PushHeaderSize)), pushHeadBuf.Bytes()...)
+		id:             tid,
-	pushMsg = append(pushMsg, data...)
+		writeData:      make(chan []byte, WriteDataChanSize),
-	pn.writeMtx.Lock()
+		writeDataSmall: make(chan []byte, WriteDataChanSize),
-	defer pn.writeMtx.Unlock()
+	}
-	if err = pn.conn.WriteMessage(websocket.BinaryMessage, pushMsg); err != nil {
+	t.initPort()
-		gLog.Printf(LvERROR, "push to %s error:%s", to, err)
+	if isClient {
-		pn.conn.Close()
+		if err = t.connect(); err != nil {
-	}
+			gLog.d("p2pTunnel connect error:%s", err)
-	return err
+			return
-}
+		}
-
+	} else {
-func (pn *P2PNetwork) read(node string, mainType uint16, subType uint16, timeout time.Duration) (head *openP2PHeader, body []byte) {
+		if err = t.listen(); err != nil {
-	var nodeID uint64
+			gLog.d("p2pTunnel listen error:%s", err)
-	if node == "" {
+			return
-		nodeID = 0
+		}
-	} else {
+	}
-		nodeID = nodeNameToID(node)
+	// store it when success
-	}
+	gLog.d("store tunnel %d", tid)
-	pn.msgMapMtx.Lock()
+	pn.allTunnels.Store(tid, t)
-	ch := pn.msgMap[nodeID]
+	return
-	pn.msgMapMtx.Unlock()
+}
-	for {
+
-		select {
+func (pn *P2PNetwork) init() error {
-		case <-time.After(timeout):
+	gLog.i("P2PNetwork init start")
-			gLog.Printf(LvERROR, "wait msg%d:%d timeout", mainType, subType)
+	defer gLog.i("P2PNetwork init end")
-			return
+	pn.initTime = time.Now()
-		case msg := <-ch:
+	pn.wgReconnect.Add(1)
-			head = &openP2PHeader{}
+	defer pn.wgReconnect.Done()
-			err := binary.Read(bytes.NewReader(msg[:openP2PHeaderSize]), binary.LittleEndian, head)
+	var err error
-			if err != nil {
+	initOK := false
-				gLog.Println(LvERROR, "read msg error:", err)
+	defer func() {
-				break
+		if !initOK {
-			}
+			// init failed, retry
-			if head.MainType != mainType || head.SubType != subType {
+			pn.close(true)
-				continue
+			gLog.e("P2PNetwork init error:%s", err)
-			}
+		}
-			if mainType == MsgPush {
+	}()
-				body = msg[openP2PHeaderSize+PushHeaderSize:]
+	ips, err := resolveServerIP(gConf.Network.ServerHost)
-			} else {
+	if err != nil {
-				body = msg[openP2PHeaderSize:]
+		gLog.e("resolve dns failed: %v", err)
-			}
+		return err
-			return
+	}
-		}
+	gConf.Network.ServerIP = ips[0]
-	}
+	if isAndroid() {
-}
+		net.DefaultResolver = &net.Resolver{
-
+			PreferGo: true,
-func (pn *P2PNetwork) updateAppHeartbeat(appID uint64) {
+			Dial: func(ctx context.Context, network, address string) (net.Conn, error) {
-	pn.apps.Range(func(id, i interface{}) bool {
+				gLog.i("lookup dns %s %s", network, address)
-		app := i.(*p2pApp)
+				dialer := &net.Dialer{
-		if app.id != appID {
+					Timeout: 5 * time.Second,
-			return true
+				}
-		}
+				primaryDNS := "119.29.29.29:53" // Tencent Cloud DNS
-		app.updateHeartbeat()
+				return dialer.DialContext(ctx, network, primaryDNS)
-		return false
+			},
-	})
+		}
-}
+	}
-
+	v4l.stop() // stop old v4 listener if exist
-func (pn *P2PNetwork) refreshIPv6(force bool) {
+	for {
-	if !force && !IsIPv6(pn.config.publicIPv6) { // not support ipv6, not refresh
+		// detect nat type
-		return
+		gConf.Network.publicIP, gConf.Network.natType, err = getNATType(gConf.Network.ServerIP, NATDetectPort1, NATDetectPort2)
-	}
+		if err != nil {
-	client := &http.Client{Timeout: time.Second * 10}
+			gLog.d("detect NAT type error:%s", err)
-	r, err := client.Get("http://6.ipw.cn")
+			break
-	if err != nil {
+		}
-		gLog.Println(LvDEBUG, "refreshIPv6 error:", err)
+		if gConf.Network.hasIPv4 == 0 && gConf.Network.hasUPNPorNATPMP == 0 { // if already has ipv4 or upnp no need test again
-		return
+			gConf.Network.hasIPv4, gConf.Network.hasUPNPorNATPMP = publicIPTest(gConf.Network.publicIP, gConf.Network.PublicIPPort)
-	}
+		}
-	defer r.Body.Close()
+
-	buf := make([]byte, 1024)
+		// for testcase
-	n, err := r.Body.Read(buf)
+		if strings.Contains(gConf.Network.Node, "openp2pS2STest") {
-	if n <= 0 {
+			gConf.Network.natType = NATSymmetric
-		gLog.Println(LvINFO, "refreshIPv6 error:", err, n)
+			gConf.Network.hasIPv4 = 0
-		return
+			gConf.Network.hasUPNPorNATPMP = 0
-	}
+			gLog.i("openp2pS2STest debug")
-	pn.config.publicIPv6 = string(buf[:n])
+
-}
+		}
+		if strings.Contains(gConf.Network.Node, "openp2pC2CTest") {
+			gConf.Network.natType = NATCone
+			gConf.Network.hasIPv4 = 0
+			gConf.Network.hasUPNPorNATPMP = 0
+			gLog.i("openp2pC2CTest debug")
+		}
+
+		// public ip and intranet connect
+		v4l.start()
+		pn.refreshIPv6()
+		gLog.i("hasIPv4:%d, UPNP:%d, NAT type:%d, publicIP:%s, IPv6:%s", gConf.Network.hasIPv4, gConf.Network.hasUPNPorNATPMP, gConf.Network.natType, gConf.Network.publicIP, gConf.IPv6())
+		gatewayURL := fmt.Sprintf("%s:%d", gConf.Network.ServerIP, gConf.Network.ServerPort)
+		uri := "/api/v1/login"
+		caCertPool, errCert := x509.SystemCertPool()
+		if errCert != nil {
+			gLog.e("Failed to load system root CAs:%s", errCert)
+			caCertPool = x509.NewCertPool()
+		}
+		caCertPool.AppendCertsFromPEM([]byte(rootCA))
+		caCertPool.AppendCertsFromPEM([]byte(rootEdgeCA))
+		caCertPool.AppendCertsFromPEM([]byte(ISRGRootX1))
+		config := tls.Config{
+			RootCAs:            caCertPool,
+			InsecureSkipVerify: gConf.TLSInsecureSkipVerify} // let's encrypt root cert "DST Root CA X3" expired at 2021/09/29. many old system(windows server 2008 etc) will not trust our cert
+		websocket.DefaultDialer.TLSClientConfig = &config
+		websocket.DefaultDialer.HandshakeTimeout = ClientAPITimeout * 3
+		u := url.URL{Scheme: "wss", Host: gatewayURL, Path: uri}
+		q := u.Query()
+		q.Add("node", gConf.Network.Node)
+		q.Add("token", fmt.Sprintf("%d", gConf.Network.Token))
+		q.Add("version", OpenP2PVersion)
+		q.Add("ipv4", gConf.Network.publicIP)
+		q.Add("ipv6", gConf.IPv6())
+		q.Add("nattype", fmt.Sprintf("%d", gConf.Network.natType))
+		q.Add("sharebandwidth", fmt.Sprintf("%d", gConf.Network.ShareBandwidth))
+		u.RawQuery = q.Encode()
+		d := websocket.Dialer{
+			NetDialContext: (&net.Dialer{Timeout: 10 * time.Second}).DialContext,
+			TLSClientConfig: &tls.Config{
+				RootCAs:            caCertPool,               // 你的根证书池
+				ServerName:         gConf.Network.ServerHost, // <--- 关键：把域名放到 ServerName
+				InsecureSkipVerify: gConf.TLSInsecureSkipVerify,
+			},
+			HandshakeTimeout: 10 * time.Second,
+		}
+
+		ws, _, err := d.Dial(u.String(), nil)
+		if err != nil {
+			gLog.e("Dial error:%s", err)
+			switch gConf.Network.ServerPort {
+			case WsPort:
+				gConf.Network.ServerPort = WsPort2
+				gLog.i("try alternative port %d", WsPort2)
+			case WsPort2:
+				gConf.Network.ServerPort = WsPort
+				gLog.i("try alternative port %d", WsPort)
+			}
+			break
+		}
+		pn.running = true
+		pn.online = true
+		pn.conn = ws
+		localAddr := strings.Split(ws.LocalAddr().String(), ":")
+		if len(localAddr) == 2 {
+			gConf.Network.localIP = localAddr[0]
+		} else {
+			gLog.e("get local ip failed:%s", ws.LocalAddr().String())
+			break
+		}
+		go pn.readLoop()
+		gConf.Network.mac = getmac(gConf.Network.localIP)
+		gConf.Network.os = getOsName()
+		go func() {
+			req := ReportBasic{
+				Mac:             gConf.Network.mac,
+				LanIP:           gConf.Network.localIP,
+				OS:              gConf.Network.os,
+				HasIPv4:         gConf.Network.hasIPv4,
+				PublicIPPort:    gConf.Network.PublicIPPort,
+				HasUPNPorNATPMP: gConf.Network.hasUPNPorNATPMP,
+				Version:         OpenP2PVersion,
+			}
+			rsp := netInfo()
+			gLog.d("netinfo:%v", rsp)
+			if rsp != nil && rsp.Country != "" {
+				if IsIPv6(rsp.IP.String()) {
+					gConf.setIPv6(rsp.IP.String())
+				}
+				req.NetInfo = *rsp
+			} else {
+				pn.refreshIPv6()
+			}
+			req.IPv6 = gConf.IPv6()
+			pn.write(MsgReport, MsgReportBasic, &req) // TODO: if report failed, many logic problems, loss lanip os version...
+			head, _ := pn.read("", MsgReport, MsgReportBasicRsp, ClientAPITimeout)
+			if head == nil {
+				gLog.e("read MsgReportBasic rsp error, retry")
+				pn.write(MsgReport, MsgReportBasic, &req) // TODO: if report failed, many logic problems, loss lanip os version...
+				head, _ := pn.read("", MsgReport, MsgReportBasicRsp, ClientAPITimeout)
+				if head == nil {
+					gLog.e("read MsgReportBasic rsp error again, exit")
+					if !isAndroid() {
+						os.Exit(9)
+					}
+				}
+				return
+			}
+		}()
+		go pn.autorunApp()
+		pn.write(MsgSDWAN, MsgSDWANInfoReq, nil)
+		initOK = true
+		gLog.d("P2PNetwork init ok")
+		break
+	}
+
+	return err
+}
+
+func (pn *P2PNetwork) handleMessage(msg []byte) {
+	head := openP2PHeader{}
+	err := binary.Read(bytes.NewReader(msg[:openP2PHeaderSize]), binary.LittleEndian, &head)
+	if err != nil {
+		gLog.e("handleMessage error:%s", err)
+		return
+	}
+	gLog.dev("handleMessage %+v", head)
+	switch head.MainType {
+	case MsgLogin:
+		// gLog.Println(LevelINFO,string(msg))
+		rsp := LoginRsp{}
+		if err = json.Unmarshal(msg[openP2PHeaderSize:], &rsp); err != nil {
+			gLog.e("wrong %v:%s", reflect.TypeOf(rsp), err)
+			return
+		}
+		if rsp.Error != 0 {
+			gLog.e("login error:%d, detail:%s", rsp.Error, rsp.Detail)
+			pn.running = false
+		} else {
+			gConf.setToken(rsp.Token)
+			gConf.setUser(rsp.User)
+			gConf.setForcev6(rsp.Forcev6 != 0)
+			if rsp.PublicIPPort != 0 {
+				gConf.Network.PublicIPPort = rsp.PublicIPPort
+			}
+			if len(rsp.Node) >= MinNodeNameLen {
+				gConf.setNode(rsp.Node)
+			}
+			gConf.save()
+			if rsp.LoginMaxDelay > 0 {
+				pn.loginMaxDelaySeconds = rsp.LoginMaxDelay
+			}
+			gLog.i("login ok. user=%s, node=%s", rsp.User, rsp.Node)
+		}
+	case MsgHeartbeat:
+		gLog.dev("P2PNetwork heartbeat ok")
+		pn.hbTime = time.Now()
+		rtt := pn.hbTime.UnixNano() - pn.t1
+		if rtt > int64(PunchTsDelay) || (pn.preRtt > 0 && rtt > pn.preRtt*5) {
+			gLog.d("rtt=%dms too large ignore", rtt/int64(time.Millisecond))
+			return // invalid hb rsp
+		}
+		pn.preRtt = rtt
+		t2 := int64(binary.LittleEndian.Uint64(msg[openP2PHeaderSize : openP2PHeaderSize+8]))
+		thisdt := pn.t1 + rtt/2 - t2
+		newdt := thisdt
+		if pn.dt != 0 {
+			ddt := thisdt - pn.dt
+			pn.ddt = ddt
+			if pn.ddtma == 0 {
+				pn.ddtma = pn.ddt
+			} else {
+				pn.ddtma = int64(float64(pn.ddtma)*(1-ma10) + float64(pn.ddt)*ma10) // avoid int64 overflow
+				newdt = pn.dt + pn.ddtma
+			}
+		}
+		pn.dt = newdt
+		gLog.dev("synctime thisdt=%dms dt=%dms ddt=%dns ddtma=%dns rtt=%dms ", thisdt/int64(time.Millisecond), pn.dt/int64(time.Millisecond), pn.ddt, pn.ddtma, rtt/int64(time.Millisecond))
+	case MsgPush:
+		handlePush(head.SubType, msg)
+	case MsgSDWAN:
+		handleSDWAN(head.SubType, msg)
+	default:
+		i, ok := pn.msgMap.Load(uint64(0))
+		if ok {
+			ch := i.(chan msgCtx)
+			select {
+			case ch <- msgCtx{data: msg, ts: time.Now()}:
+			default:
+				gLog.e("msgQueue full, drop it")
+			}
+		}
+
+		return
+	}
+}
+
+func (pn *P2PNetwork) readLoop() {
+	gLog.d("P2PNetwork readLoop start")
+	pn.wgReconnect.Add(1)
+	defer pn.wgReconnect.Done()
+	ctx, cancel := context.WithCancel(context.Background())
+	defer cancel()
+
+	// 使用带超时的 goroutine 读取
+	readChan := make(chan []byte, 10)
+
+	go func() {
+		for {
+			select {
+			case <-ctx.Done():
+				return
+			default:
+				pn.conn.SetReadDeadline(time.Now().Add(NetworkHeartbeatTime + 10*time.Second))
+				_, msg, err := pn.conn.ReadMessage()
+				if err != nil {
+					gLog.e("ReadMessage error:%s", err)
+					readChan <- nil
+					return
+				}
+				readChan <- msg
+			}
+		}
+	}()
+
+	readTimeout := 60 * time.Second
+
+	for pn.running {
+		select {
+		case result := <-readChan:
+			if result == nil {
+				// 处理错误
+				pn.close(false)
+				return
+			}
+			pn.handleMessage(result)
+
+		case <-time.After(readTimeout):
+			gLog.e("ReadMessage timeout after %v", readTimeout)
+			cancel()
+			pn.close(false)
+			return
+		}
+	}
+	gLog.d("P2PNetwork readLoop end")
+}
+
+func (pn *P2PNetwork) write(mainType uint16, subType uint16, packet interface{}) error {
+	if !pn.online {
+		return errors.New("P2P network offline")
+	}
+	msg, err := newMessage(mainType, subType, packet)
+	if err != nil {
+		return err
+	}
+	pn.writeMtx.Lock()
+	defer pn.writeMtx.Unlock()
+	pn.conn.SetWriteDeadline(time.Now().Add(NetworkHeartbeatTime))
+	if err = pn.conn.WriteMessage(websocket.BinaryMessage, msg); err != nil {
+		gLog.e("write msgType %d,%d error:%s", mainType, subType, err)
+		pn.close(false)
+	}
+	return err
+}
+
+func (pn *P2PNetwork) relay(to uint64, body []byte) error {
+	i, ok := pn.allTunnels.Load(to)
+	if !ok {
+		return ErrRelayTunnelNotFound
+	}
+	tunnel := i.(*P2PTunnel)
+	if tunnel.config.shareBandwidth > 0 {
+		pn.limiter.Add(len(body), true)
+	}
+	var err error
+	if err = tunnel.conn.WriteBuffer(body); err != nil {
+		gLog.dev("relay to %d len=%d error:%s", to, len(body), err)
+	}
+	return err
+}
+
+func (pn *P2PNetwork) push(to string, subType uint16, packet interface{}) error {
+	// gLog.d("push msgType %d to %s", subType, to)
+	if !pn.online {
+		return errors.New("client offline")
+	}
+	pushHead := PushHeader{}
+	pushHead.From = gConf.nodeID()
+	pushHead.To = NodeNameToID(to)
+	pushHeadBuf := new(bytes.Buffer)
+	err := binary.Write(pushHeadBuf, binary.LittleEndian, pushHead)
+	if err != nil {
+		return err
+	}
+	data, err := json.Marshal(packet)
+	if err != nil {
+		return err
+	}
+	// gLog.Println(LevelINFO,"write packet:", string(data))
+	pushMsg := append(encodeHeader(MsgPush, subType, uint32(len(data)+PushHeaderSize)), pushHeadBuf.Bytes()...)
+	pushMsg = append(pushMsg, data...)
+	pn.writeMtx.Lock()
+	defer pn.writeMtx.Unlock()
+	pn.conn.SetWriteDeadline(time.Now().Add(NetworkHeartbeatTime))
+	if err = pn.conn.WriteMessage(websocket.BinaryMessage, pushMsg); err != nil {
+		gLog.e("push to %s error:%s", to, err)
+		pn.close(false)
+	}
+	return err
+}
+
+func (pn *P2PNetwork) close(isRestartDelay bool) {
+	if pn.running {
+		if pn.conn != nil {
+			pn.conn.Close()
+		}
+		pn.running = false
+	}
+	select {
+	case pn.restartCh <- isRestartDelay:
+	default:
+	}
+}
+
+func (pn *P2PNetwork) read(node string, mainType uint16, subType uint16, timeout time.Duration) (head *openP2PHeader, body []byte) {
+	var nodeID uint64
+	if node == "" {
+		nodeID = 0
+	} else {
+		nodeID = NodeNameToID(node)
+	}
+	i, ok := pn.msgMap.Load(nodeID)
+	if !ok {
+		gLog.e("read msg error: %s not found", node)
+		return
+	}
+	ch := i.(chan msgCtx)
+	for {
+		select {
+		case <-time.After(timeout):
+			gLog.e("read msg error %d:%d timeout", mainType, subType)
+			return
+		case msg := <-ch:
+			head = &openP2PHeader{}
+			err := binary.Read(bytes.NewReader(msg.data[:openP2PHeaderSize]), binary.LittleEndian, head)
+			if err != nil {
+				gLog.e("read msg error:%s", err)
+				break
+			}
+			if time.Since(msg.ts) > ReadMsgTimeout {
+				gLog.d("read msg error expired %d:%d", head.MainType, head.SubType)
+				continue
+			}
+			if head.MainType != mainType || head.SubType != subType {
+				// gLog.d("read msg error type %d:%d expect %d:%d, requeue it", head.MainType, head.SubType, mainType, subType)
+				ch <- msg
+				time.Sleep(time.Millisecond * 50)
+				continue
+			}
+			if mainType == MsgPush {
+				body = msg.data[openP2PHeaderSize+PushHeaderSize:]
+			} else {
+				body = msg.data[openP2PHeaderSize:]
+			}
+			return
+		}
+	}
+}
+
+func (pn *P2PNetwork) updateAppHeartbeat(appID uint64, rtid uint64, updateRelayTs bool) {
+	pn.apps.Range(func(id, i interface{}) bool {
+		app := i.(*p2pApp)
+		if app.id == appID {
+			if updateRelayTs {
+				app.UpdateRelayHeartbeatTs(rtid)
+			} else {
+				app.UpdateHeartbeat(rtid)
+			}
+
+		}
+		return true
+	})
+}
+
+// ipv6 will expired need to refresh.
+func (pn *P2PNetwork) refreshIPv6() {
+
+	for i := 0; i < 2; i++ {
+		url := "http://ipv6.ddnspod.com/"
+		if i == 1 {
+			url = "ipv6.icanhazip.com"
+		}
+		client := &http.Client{Timeout: time.Second * 10}
+		r, err := client.Get(url)
+		if err != nil {
+			gLog.d("refreshIPv6 error:%s", err)
+			continue
+		}
+		defer r.Body.Close()
+		buf := make([]byte, 1024)
+		n, err := r.Body.Read(buf)
+		if n <= 0 {
+			gLog.e("refreshIPv6 error:%s", err)
+			continue
+		}
+		if IsIPv6(string(buf[:n])) {
+			newIPv6 := string(buf[:n])
+			if newIPv6 != gConf.IPv6() {
+				gLog.i("refreshIPv6 change:%s ---> %s", gConf.IPv6(), newIPv6)
+				gConf.setIPv6(newIPv6)
+
+			}
+		}
+		gLog.d("refreshIPv6:%s", gConf.IPv6())
+		break
+	}
+
+}
+
+func (pn *P2PNetwork) requestPeerInfo(config *AppConfig) error {
+	// request peer info
+	// TODO: multi-thread issue
+	pn.reqGatewayMtx.Lock()
+	pn.write(MsgQuery, MsgQueryPeerInfoReq, &QueryPeerInfoReq{config.peerToken, config.PeerNode})
+	head, body := pn.read("", MsgQuery, MsgQueryPeerInfoRsp, ClientAPITimeout)
+	pn.reqGatewayMtx.Unlock()
+	if head == nil {
+		gLog.e("requestPeerInfo error")
+		return ErrNetwork // network error, should not be ErrPeerOffline
+	}
+	rsp := QueryPeerInfoRsp{}
+	if err := json.Unmarshal(body, &rsp); err != nil {
+		return ErrMsgFormat
+	}
+	if rsp.Online == 0 {
+		return ErrPeerOffline
+	}
+	if compareVersion(rsp.Version, LeastSupportVersion) < 0 {
+		return ErrVersionNotCompatible
+	}
+	config.peerVersion = rsp.Version
+	config.peerLanIP = rsp.LanIP
+	config.hasIPv4 = rsp.HasIPv4
+	config.peerIP = rsp.IPv4
+	config.peerIPv6 = rsp.IPv6
+	config.hasUPNPorNATPMP = rsp.HasUPNPorNATPMP
+	config.peerNatType = rsp.NatType
+	///
+	return nil
+}
+
+func (pn *P2PNetwork) StartSDWAN() {
+	// request peer info
+	pn.sdwan = &p2pSDWAN{}
+}
+
+func (pn *P2PNetwork) ConnectNode(node string) error {
+	if gConf.nodeID() <= NodeNameToID(node) {
+		return errors.New("only the bigger nodeid connect")
+	}
+	peerNodeID := fmt.Sprintf("%d", NodeNameToID(node))
+	config := AppConfig{Enabled: 1}
+	config.AppName = peerNodeID
+	config.SrcPort = 0
+	config.PeerNode = node
+	sdwan := gConf.getSDWAN()
+	config.PunchPriority = int(sdwan.PunchPriority)
+	// config.UnderlayProtocol = "kcp"
+	if node != sdwan.CentralNode && gConf.Network.Node != sdwan.CentralNode { // neither is centralnode
+		config.RelayNode = sdwan.CentralNode
+		config.ForceRelay = int(sdwan.ForceRelay)
+		if sdwan.Mode == SDWANModeCentral {
+			config.ForceRelay = 1
+		}
+	}
+
+	gConf.add(config, true)
+	return nil
+}
+
+func (pn *P2PNetwork) WriteNode(nodeID uint64, buff []byte) error {
+	i, ok := pn.apps.Load(nodeID)
+	if !ok {
+		return errors.New("peer not found")
+	}
+	var err error
+	app := i.(*p2pApp)
+	// TODO: move to app.write
+	err = app.WriteNodeDataMP(buff)
+	if err != nil {
+		gLog.dev("appID:%d WriteNodeDataMP %s", app.id, err)
+	}
+	// gLog.dev("%d tunnel write node data bodylen=%d, relay=%t", app.Tunnel().id, len(buff), !app.isDirect())
+	// if app.DirectTunnel() != nil { // direct
+	// 	app.Tunnel().asyncWriteNodeData(MsgP2P, MsgNodeData, buff)
+	// }
+	// if app.Tunnel() != nil { // relay
+	// 	fromNodeIDHead := new(bytes.Buffer)
+	// 	binary.Write(fromNodeIDHead, binary.LittleEndian, gConf.nodeID())
+	// 	all := app.RelayHead().Bytes()
+	// 	all = append(all, encodeHeader(MsgP2P, MsgRelayNodeData, uint32(len(buff)+overlayHeaderSize))...)
+	// 	all = append(all, fromNodeIDHead.Bytes()...)
+	// 	all = append(all, buff...)
+	// 	app.Tunnel().asyncWriteNodeData(MsgP2P, MsgRelayData, all)
+	// }
+
+	return err
+}
+
+func (pn *P2PNetwork) WriteBroadcast(buff []byte) error {
+	///
+	pn.apps.Range(func(id, i interface{}) bool {
+		// newDestIP := net.ParseIP("10.2.3.2")
+		// copy(buff[16:20], newDestIP.To4())
+		// binary.BigEndian.PutUint16(buff[10:12], 0) // set checksum=0 for calc checksum
+		// ipChecksum := calculateChecksum(buff[0:20])
+		// binary.BigEndian.PutUint16(buff[10:12], ipChecksum)
+		// binary.BigEndian.PutUint16(buff[26:28], 0x082e)
+		app := i.(*p2pApp)
+
+		if app.config.SrcPort != 0 { // normal portmap app
+			return true
+		}
+		if app.config.peerIP == gConf.Network.publicIP { // mostly in a lan
+			return true
+		}
+		err := app.WriteNodeDataMP(buff)
+		if err != nil {
+			gLog.dev("appID:%d WriteNodeDataMP %s", app.id, err)
+		}
+		return true
+	})
+	return nil
+}
+
+func (pn *P2PNetwork) ReadNode(tm time.Duration) []byte {
+	select {
+	case nd := <-pn.nodeData:
+		return nd
+	case <-time.After(tm):
+	}
+	return nil
+}

core/p2ptunnel.go

@@ -1,621 +1,987 @@
-package openp2p
+package openp2p
-
+
-import (
+import (
-	"bytes"
+	"bytes"
-	"encoding/binary"
+	"context"
-	"encoding/json"
+	"encoding/binary"
-	"errors"
+	"encoding/json"
-	"fmt"
+	"errors"
-	"math/rand"
+	"fmt"
-	"net"
+	"math/rand"
-	"sync"
+	"net"
-	"time"
+	"reflect"
-)
+	"sync"
-
+	"sync/atomic"
-type P2PTunnel struct {
+	"time"
-	pn            *P2PNetwork
+)
-	conn          underlay
+
-	hbTime        time.Time
+const WriteDataChanSize int = 8192
-	hbMtx         sync.Mutex
+
-	hbTimeRelay   time.Time
+var buildTunnelMtx sync.Mutex
-	config        AppConfig
+
-	la            *net.UDPAddr // local hole address
+const (
-	ra            *net.UDPAddr // remote hole address
+	StatusIdle    = 0
-	overlayConns  sync.Map     // both TCP and UDP
+	StatusWriting = 1
-	id            uint64
+)
-	running       bool
+
-	runMtx        sync.Mutex
+type P2PTunnel struct {
-	tunnelServer  bool // different from underlayServer
+	conn           underlay
-	coneLocalPort int
+	hbTime         time.Time
-	coneNatPort   int
+	hbMtx          sync.Mutex
-	linkModeWeb   string // use config.linkmode
+	whbTime        time.Time
-}
+	config         AppConfig
-
+	localHoleAddr  *net.UDPAddr // local hole address
-func (t *P2PTunnel) requestPeerInfo() error {
+	remoteHoleAddr *net.UDPAddr // remote hole address
-	// request peer info
+	id             uint64       // client side alloc rand.uint64 = server side
-	t.pn.write(MsgQuery, MsgQueryPeerInfoReq, &QueryPeerInfoReq{t.config.peerToken, t.config.PeerNode})
+
-	head, body := t.pn.read("", MsgQuery, MsgQueryPeerInfoRsp, time.Second*10)
+	running        bool
-	if head == nil {
+	runMtx         sync.Mutex
-		return ErrPeerOffline
+	coneLocalPort  int
-	}
+	coneNatPort    int
-	rsp := QueryPeerInfoRsp{}
+	linkModeWeb    string // use config.linkmode
-	err := json.Unmarshal(body, &rsp)
+	punchTs        uint64
-	if err != nil {
+	writeData      chan []byte
-		gLog.Printf(LvERROR, "wrong QueryPeerInfoRsp:%s", err)
+	writeDataSmall chan []byte
-		return ErrMsgFormat
+}
-	}
+
-	if rsp.Online == 0 {
+func (t *P2PTunnel) initPort() {
-		return ErrPeerOffline
+	t.running = true
-	}
+	localPort := int(rand.Uint32()%8192 + 1025) // if the process has bug, will add many upnp port. use specify p2p port by param
-	if compareVersion(rsp.Version, LeastSupportVersion) == LESS {
+	if t.config.linkMode == LinkModeTCP6 || t.config.linkMode == LinkModeTCP4 || t.config.linkMode == LinkModeUDP4 || t.config.linkMode == LinkModeIntranet {
-		return ErrVersionNotCompatible
+		t.coneLocalPort = gConf.Network.PublicIPPort
-	}
+		t.coneNatPort = gConf.Network.PublicIPPort // symmetric doesn't need coneNatPort
-	t.config.peerVersion = rsp.Version
+	}
-	t.config.hasIPv4 = rsp.HasIPv4
+	if t.config.linkMode == LinkModeUDPPunch {
-	t.config.peerIP = rsp.IPv4
+		// prepare one random cone hole manually
-	t.config.peerIPv6 = rsp.IPv6
+		_, natPort, _ := natDetectUDP(gConf.Network.ServerIP, NATDetectPort1, localPort)
-	t.config.hasUPNPorNATPMP = rsp.HasUPNPorNATPMP
+		t.coneLocalPort = localPort
-	t.config.peerNatType = rsp.NatType
+		t.coneNatPort = natPort
-	///
+	}
-	return nil
+	if t.config.linkMode == LinkModeTCPPunch {
-}
+		// prepare one random cone hole by system automatically
-func (t *P2PTunnel) initPort() {
+		_, natPort, localPort2, _ := natDetectTCP(gConf.Network.ServerIP, NATDetectPort1, 0)
-	t.running = true
+		t.coneLocalPort = localPort2
-	t.hbMtx.Lock()
+		t.coneNatPort = natPort
-	t.hbTime = time.Now()
+	}
-	t.hbMtx.Unlock()
+	if t.config.linkMode == LinkModeTCP6 && compareVersion(t.config.peerVersion, IPv6PunchVersion) >= 0 {
-	t.hbTimeRelay = time.Now().Add(time.Second * 600) // TODO: test fake time
+		t.coneLocalPort = localPort
-	localPort := int(rand.Uint32()%15000 + 50000)     // if the process has bug, will add many upnp port. use specify p2p port by param
+		t.coneNatPort = localPort
-	if t.config.linkMode == LinkModeTCP6 {
+	}
-		t.pn.refreshIPv6(false)
+	t.localHoleAddr = &net.UDPAddr{IP: net.ParseIP(gConf.Network.localIP), Port: t.coneLocalPort}
-	}
+	gLog.d("prepare punching port %d:%d", t.coneLocalPort, t.coneNatPort)
-	if t.config.linkMode == LinkModeTCP6 || t.config.linkMode == LinkModeTCP4 {
+}
-		t.coneLocalPort = t.pn.config.TCPPort
+
-		t.coneNatPort = t.pn.config.TCPPort // symmetric doesn't need coneNatPort
+func (t *P2PTunnel) connect() error {
-	}
+	gLog.d("start p2pTunnel to %s ", t.config.LogPeerNode())
-	if t.config.linkMode == LinkModeUDPPunch {
+	appKey := uint64(0)
-		// prepare one random cone hole
+	req := PushConnectReq{
-		_, natPort, _ := natTest(t.pn.config.ServerHost, t.pn.config.UDPPort1, localPort)
+		Token:            t.config.peerToken,
-		t.coneLocalPort = localPort
+		From:             gConf.Network.Node,
-		t.coneNatPort = natPort
+		FromIP:           gConf.Network.publicIP,
-	}
+		ConeNatPort:      t.coneNatPort,
-	if t.config.linkMode == LinkModeTCPPunch {
+		NatType:          gConf.Network.natType,
-		// prepare one random cone hole
+		HasIPv4:          gConf.Network.hasIPv4,
-		_, natPort := natTCP(t.pn.config.ServerHost, IfconfigPort1, localPort)
+		IPv6:             gConf.IPv6(),
-		t.coneLocalPort = localPort
+		HasUPNPorNATPMP:  gConf.Network.hasUPNPorNATPMP,
-		t.coneNatPort = natPort
+		ID:               t.id,
-	}
+		AppKey:           appKey,
-	t.la = &net.UDPAddr{IP: net.ParseIP(t.pn.config.localIP), Port: t.coneLocalPort}
+		Version:          OpenP2PVersion,
-	gLog.Printf(LvDEBUG, "prepare punching port %d:%d", t.coneLocalPort, t.coneNatPort)
+		LinkMode:         t.config.linkMode,
-}
+		IsUnderlayServer: t.config.isUnderlayServer ^ 1, // peer
-
+		UnderlayProtocol: t.config.UnderlayProtocol,
-func (t *P2PTunnel) connect() error {
+	}
-	gLog.Printf(LvDEBUG, "start p2pTunnel to %s ", t.config.PeerNode)
+	if req.Token == 0 { // no relay token
-	t.tunnelServer = false
+		req.Token = gConf.Network.Token
-	appKey := uint64(0)
+	}
-	req := PushConnectReq{
+	GNetwork.push(t.config.PeerNode, MsgPushConnectReq, req)
-		Token:            t.config.peerToken,
+	head, body := GNetwork.read(t.config.PeerNode, MsgPush, MsgPushConnectRsp, UnderlayConnectTimeout*3)
-		From:             t.pn.config.Node,
+	if head == nil {
-		FromIP:           t.pn.config.publicIP,
+		return errors.New("connect error")
-		ConeNatPort:      t.coneNatPort,
+	}
-		NatType:          t.pn.config.natType,
+	rsp := PushConnectRsp{}
-		HasIPv4:          t.pn.config.hasIPv4,
+	if err := json.Unmarshal(body, &rsp); err != nil {
-		IPv6:             t.pn.config.publicIPv6,
+		gLog.e("wrong %v:%s", reflect.TypeOf(rsp), err)
-		HasUPNPorNATPMP:  t.pn.config.hasUPNPorNATPMP,
+		return err
-		ID:               t.id,
+	}
-		AppKey:           appKey,
+	// gLog.Println(LevelINFO, rsp)
-		Version:          OpenP2PVersion,
+	if rsp.Error != 0 {
-		LinkMode:         t.config.linkMode,
+		return errors.New(rsp.Detail)
-		IsUnderlayServer: t.config.isUnderlayServer ^ 1,
+	}
-	}
+	t.config.peerNatType = rsp.NatType
-	if req.Token == 0 { // no relay token
+	t.config.hasIPv4 = rsp.HasIPv4
-		req.Token = t.pn.config.Token
+	t.config.peerIPv6 = rsp.IPv6
-	}
+	t.config.hasUPNPorNATPMP = rsp.HasUPNPorNATPMP
-	t.pn.push(t.config.PeerNode, MsgPushConnectReq, req)
+	t.config.peerVersion = rsp.Version
-	head, body := t.pn.read(t.config.PeerNode, MsgPush, MsgPushConnectRsp, time.Second*10)
+	t.config.peerConeNatPort = rsp.ConeNatPort
-	if head == nil {
+	t.config.peerIP = rsp.FromIP
-		return errors.New("connect error")
+	t.punchTs = rsp.PunchTs
-	}
+	err := t.start()
-	rsp := PushConnectRsp{}
+	if err != nil {
-	err := json.Unmarshal(body, &rsp)
+		gLog.d("handshake error:%s", err)
-	if err != nil {
+	}
-		gLog.Printf(LvERROR, "wrong MsgPushConnectRsp:%s", err)
+	return err
-		return err
+}
-	}
+
-	// gLog.Println(LevelINFO, rsp)
+func (t *P2PTunnel) isRuning() bool {
-	if rsp.Error != 0 {
+	t.runMtx.Lock()
-		return errors.New(rsp.Detail)
+	defer t.runMtx.Unlock()
-	}
+	return t.running
-	t.config.peerNatType = rsp.NatType
+}
-	t.config.hasIPv4 = rsp.HasIPv4
+
-	t.config.peerIPv6 = rsp.IPv6
+func (t *P2PTunnel) setRun(running bool) {
-	t.config.hasUPNPorNATPMP = rsp.HasUPNPorNATPMP
+	t.runMtx.Lock()
-	t.config.peerVersion = rsp.Version
+	defer t.runMtx.Unlock()
-	t.config.peerConeNatPort = rsp.ConeNatPort
+	t.running = running
-	t.config.peerIP = rsp.FromIP
+}
-	err = t.start()
+
-	if err != nil {
+func (t *P2PTunnel) isActive() bool {
-		gLog.Println(LvERROR, "handshake error:", err)
+	if !t.isRuning() || t.conn == nil {
-		err = ErrorHandshake
+		return false
-	}
+	}
-	return err
+	t.hbMtx.Lock()
-}
+	defer t.hbMtx.Unlock()
-
+	res := time.Now().Before(t.hbTime.Add(TunnelHeartbeatTime * 2))
-func (t *P2PTunnel) isRuning() bool {
+	if !res {
-	t.runMtx.Lock()
+		gLog.d("%d tunnel isActive false", t.id)
-	defer t.runMtx.Unlock()
+	}
-	return t.running
+	return res
-}
+}
-
+
-func (t *P2PTunnel) setRun(running bool) {
+func (t *P2PTunnel) checkActive() bool {
-	t.runMtx.Lock()
+	if !t.isActive() {
-	defer t.runMtx.Unlock()
+		return false
-	t.running = running
+	}
-}
+	hbt := time.Now()
-
+	t.conn.WriteBytes(MsgP2P, MsgTunnelHeartbeat, nil)
-func (t *P2PTunnel) isActive() bool {
+	isActive := false
-	t.hbMtx.Lock()
+	// wait at most 5s
-	defer t.hbMtx.Unlock()
+	for i := 0; i < 50 && !isActive; i++ {
-	return time.Now().Before(t.hbTime.Add(TunnelIdleTimeout))
+		t.hbMtx.Lock()
-}
+		if t.hbTime.After(hbt) {
-
+			isActive = true
-func (t *P2PTunnel) checkActive() bool {
+		}
-	hbt := time.Now()
+		t.hbMtx.Unlock()
-	t.hbMtx.Lock()
+		time.Sleep(time.Millisecond * 100)
-	if t.hbTime.Before(time.Now().Add(-TunnelHeartbeatTime)) {
+	}
-		t.hbMtx.Unlock()
+	gLog.d("checkActive %t. hbtime=%d", isActive, t.hbTime)
-		return false
+	return isActive
-	}
+}
-	t.hbMtx.Unlock()
+
-	// hbtime within TunnelHeartbeatTime, check it now
+// call when user delete tunnel
-	t.conn.WriteBytes(MsgP2P, MsgTunnelHeartbeat, nil)
+func (t *P2PTunnel) close() {
-	isActive := false
+	GNetwork.NotifyTunnelClose(t)
-	// wait at most 5s
+	if !t.running {
-	for i := 0; i < 50 && !isActive; i++ {
+		return
-		t.hbMtx.Lock()
+	}
-		if t.hbTime.After(hbt) {
+	t.setRun(false)
-			isActive = true
+	if t.conn != nil {
-		}
+		t.conn.Close()
-		t.hbMtx.Unlock()
+	}
-		time.Sleep(time.Millisecond * 100)
+	GNetwork.allTunnels.Delete(t.id)
-	}
+	gLog.i("%d p2ptunnel close %s ", t.id, t.config.LogPeerNode())
-	return isActive
+}
-}
+
-
+func (t *P2PTunnel) start() error {
-// call when user delete tunnel
+	if t.config.linkMode == LinkModeUDPPunch {
-func (t *P2PTunnel) close() {
+		if err := t.handshake(); err != nil {
-	t.setRun(false)
+			return err
-	t.pn.allTunnels.Delete(t.id)
+		}
-}
+	}
-
+	err := t.connectUnderlay()
-func (t *P2PTunnel) start() error {
+	if err != nil {
-	if t.config.linkMode == LinkModeUDPPunch {
+		gLog.d("connectUnderlay error:%s", err)
-		if err := t.handshake(); err != nil {
+		return err
-			return err
+	}
-		}
+	return nil
-	}
+}
-	err := t.connectUnderlay()
+
-	if err != nil {
+func (t *P2PTunnel) handshake() error {
-		gLog.Println(LvERROR, err)
+	if t.config.peerConeNatPort > 0 { // only peer is cone should prepare t.ra
-		return err
+		var err error
-	}
+		t.remoteHoleAddr, err = net.ResolveUDPAddr("udp", fmt.Sprintf("%s:%d", t.config.peerIP, t.config.peerConeNatPort))
-	return nil
+		if err != nil {
-}
+			return err
-
+		}
-func (t *P2PTunnel) handshake() error {
+	}
-	if t.config.peerConeNatPort > 0 { // only peer is cone should prepare t.ra
+	if compareVersion(t.config.peerVersion, SyncServerTimeVersion) < 0 {
-		var err error
+		gLog.d("peer version %s less than %s", t.config.peerVersion, SyncServerTimeVersion)
-		t.ra, err = net.ResolveUDPAddr("udp", fmt.Sprintf("%s:%d", t.config.peerIP, t.config.peerConeNatPort))
+	} else {
-		if err != nil {
+		ts := time.Duration(int64(t.punchTs) + GNetwork.dt + GNetwork.ddtma*int64(time.Since(GNetwork.hbTime)+PunchTsDelay)/int64(NetworkHeartbeatTime) - time.Now().UnixNano())
-			return err
+		if ts > PunchTsDelay || ts < 0 {
-		}
+			ts = PunchTsDelay
-	}
+		}
-	gLog.Println(LvDEBUG, "handshake to ", t.config.PeerNode)
+		gLog.d("sleep %d ms", ts/time.Millisecond)
-	var err error
+		time.Sleep(ts)
-	// TODO: handle NATNone, nodes with public ip has no punching
+	}
-	if t.pn.config.natType == NATCone && t.config.peerNatType == NATCone {
+	gLog.d("handshake to %s", t.config.LogPeerNode())
-		err = handshakeC2C(t)
+	var err error
-	} else if t.config.peerNatType == NATSymmetric && t.pn.config.natType == NATSymmetric {
+	if gConf.Network.natType == NATCone && t.config.peerNatType == NATCone {
-		err = ErrorS2S
+		err = handshakeC2C(t)
-		t.close()
+	} else if t.config.peerNatType == NATSymmetric && gConf.Network.natType == NATSymmetric {
-	} else if t.config.peerNatType == NATSymmetric && t.pn.config.natType == NATCone {
+		err = ErrorS2S
-		err = handshakeC2S(t)
+		t.close()
-	} else if t.config.peerNatType == NATCone && t.pn.config.natType == NATSymmetric {
+	} else if t.config.peerNatType == NATSymmetric && gConf.Network.natType == NATCone {
-		err = handshakeS2C(t)
+		err = handshakeC2S(t)
-	} else {
+	} else if t.config.peerNatType == NATCone && gConf.Network.natType == NATSymmetric {
-		return errors.New("unknown error")
+		err = handshakeS2C(t)
-	}
+	} else {
-	if err != nil {
+		return errors.New("unknown error")
-		gLog.Println(LvERROR, "punch handshake error:", err)
+	}
-		return err
+	if err != nil {
-	}
+		gLog.d("punch handshake error:%s", err)
-	gLog.Printf(LvDEBUG, "handshake to %s ok", t.config.PeerNode)
+		return err
-	return nil
+	}
-}
+	gLog.d("handshake to %s ok", t.config.LogPeerNode())
-
+	return nil
-func (t *P2PTunnel) connectUnderlay() (err error) {
+}
-	switch t.config.linkMode {
+
-	case LinkModeTCP6:
+func (t *P2PTunnel) connectUnderlay() (err error) {
-		t.conn, err = t.connectUnderlayTCP6()
+	switch t.config.linkMode {
-	case LinkModeTCP4:
+	case LinkModeTCP6:
-		t.conn, err = t.connectUnderlayTCP()
+		if compareVersion(t.config.peerVersion, IPv6PunchVersion) >= 0 {
-	case LinkModeTCPPunch:
+			t.conn, err = t.connectUnderlayTCP()
-		t.conn, err = t.connectUnderlayTCP()
+		} else {
-	case LinkModeUDPPunch:
+			t.conn, err = t.connectUnderlayTCP6()
-		t.conn, err = t.connectUnderlayQuic()
+		}
-
+	case LinkModeTCP4:
-	}
+		t.conn, err = t.connectUnderlayTCP()
-	if err != nil {
+	case LinkModeUDP4:
-		return err
+		t.conn, err = t.connectUnderlayUDP()
-	}
+	case LinkModeTCPPunch:
-	if t.conn == nil {
+		if gConf.Network.natType == NATSymmetric || t.config.peerNatType == NATSymmetric {
-		return errors.New("connect underlay error")
+			t.conn, err = t.connectUnderlayTCPSymmetric()
-	}
+		} else {
-	t.setRun(true)
+			t.conn, err = t.connectUnderlayTCP()
-	go t.readLoop()
+		}
-	go t.heartbeatLoop()
+	case LinkModeIntranet:
-	return nil
+		t.conn, err = t.connectUnderlayTCP()
-}
+	case LinkModeUDPPunch:
-
+		t.conn, err = t.connectUnderlayUDP()
-func (t *P2PTunnel) connectUnderlayQuic() (c underlay, err error) {
+
-	gLog.Println(LvINFO, "connectUnderlayQuic start")
+	}
-	defer gLog.Println(LvINFO, "connectUnderlayQuic end")
+	if err != nil {
-	var qConn *underlayQUIC
+		return err
-	if t.config.isUnderlayServer == 1 {
+	}
-		time.Sleep(time.Millisecond * 10) // punching udp port will need some times in some env
+	if t.conn == nil {
-		qConn, err = listenQuic(t.la.String(), TunnelIdleTimeout)
+		return errors.New("connect underlay error")
-		if err != nil {
+	}
-			gLog.Println(LvINFO, "listen quic error:", err, ", retry...")
+	t.setRun(true)
-		}
+	go t.readLoop()
-		t.pn.push(t.config.PeerNode, MsgPushUnderlayConnect, nil)
+	go t.writeLoop()
-		err = qConn.Accept()
+	return nil
-		if err != nil {
+}
-			qConn.CloseListener()
+
-			return nil, fmt.Errorf("accept quic error:%s", err)
+func (t *P2PTunnel) connectUnderlayUDP() (c underlay, err error) {
-		}
+	gLog.d("connectUnderlayUDP %s start ", t.config.LogPeerNode())
-		_, buff, err := qConn.ReadBuffer()
+	defer gLog.d("connectUnderlayUDP %s end ", t.config.LogPeerNode())
-		if err != nil {
+	var ul underlay
-			qConn.listener.Close()
+	underlayProtocol := t.config.UnderlayProtocol
-			return nil, fmt.Errorf("read start msg error:%s", err)
+	if underlayProtocol == "" {
-		}
+		underlayProtocol = "quic"
-		if buff != nil {
+	}
-			gLog.Println(LvDEBUG, string(buff))
+	if t.config.isUnderlayServer == 1 {
-		}
+		// TODO: move to a func
-		qConn.WriteBytes(MsgP2P, MsgTunnelHandshakeAck, []byte("OpenP2P,hello2"))
+		time.Sleep(time.Millisecond * 10) // punching udp port will need some times in some env
-		gLog.Println(LvDEBUG, "quic connection ok")
+		go GNetwork.push(t.config.PeerNode, MsgPushUnderlayConnect, nil)
-		return qConn, nil
+		if t.config.linkMode == LinkModeUDP4 {
-	}
+			if v4l != nil {
-
+				ul = v4l.getUnderlay(t.id)
-	//else
+			}
-	conn, e := net.ListenUDP("udp", t.la)
+			if ul == nil {
-	if e != nil {
+				return nil, fmt.Errorf("listen UDP4 error")
-		time.Sleep(time.Millisecond * 10)
+			}
-		conn, e = net.ListenUDP("udp", t.la)
+			gLog.d("UDP4 connection ok")
-		if e != nil {
+		} else {
-			return nil, fmt.Errorf("quic listen error:%s", e)
+			if t.config.UnderlayProtocol == "kcp" {
-		}
+				// ul, err = listenKCP(t.localHoleAddr.String(), TunnelIdleTimeout)
-	}
+			} else {
-	t.pn.read(t.config.PeerNode, MsgPush, MsgPushUnderlayConnect, time.Second*5)
+				ul, err = listenQuic(t.localHoleAddr.String(), TunnelIdleTimeout)
-	gLog.Println(LvDEBUG, "quic dial to ", t.ra.String())
+			}
-	qConn, e = dialQuic(conn, t.ra, TunnelIdleTimeout)
+		}
-	if e != nil {
+
-		return nil, fmt.Errorf("quic dial to %s error:%s", t.ra.String(), e)
+		if err != nil {
-	}
+			gLog.i("listen %s error:%s", underlayProtocol, err)
-	handshakeBegin := time.Now()
+			return nil, err
-	qConn.WriteBytes(MsgP2P, MsgTunnelHandshake, []byte("OpenP2P,hello"))
+		}
-	_, buff, err := qConn.ReadBuffer()
+
-	if e != nil {
+		_, buff, err := ul.ReadBuffer()
-		qConn.listener.Close()
+		if err != nil {
-		return nil, fmt.Errorf("read MsgTunnelHandshake error:%s", err)
+			ul.Close()
-	}
+			return nil, fmt.Errorf("read start msg error:%s", err)
-	if buff != nil {
+		}
-		gLog.Println(LvDEBUG, string(buff))
+		if buff != nil {
-	}
+			gLog.d("handshake flag:%s", string(buff))
-
+		}
-	gLog.Println(LvINFO, "rtt=", time.Since(handshakeBegin))
+		ul.WriteBytes(MsgP2P, MsgTunnelHandshakeAck, []byte("OpenP2P,hello2"))
-	gLog.Println(LvDEBUG, "quic connection ok")
+		gLog.d("%s connection ok", underlayProtocol)
-	t.linkModeWeb = LinkModeUDPPunch
+		return ul, nil
-	return qConn, nil
+	}
-}
+
-
+	//client side
-// websocket
+	listenAddr := t.localHoleAddr
-func (t *P2PTunnel) connectUnderlayTCP() (c underlay, err error) {
+	if t.config.linkMode == LinkModeUDP4 {
-	gLog.Println(LvINFO, "connectUnderlayTCP start")
+		listenAddr = &net.UDPAddr{IP: net.ParseIP(gConf.Network.localIP), Port: 0}
-	defer gLog.Println(LvINFO, "connectUnderlayTCP end")
+		t.remoteHoleAddr, err = net.ResolveUDPAddr("udp", fmt.Sprintf("%s:%d", t.config.peerIP, t.config.peerConeNatPort))
-	var qConn *underlayTCP
+		if err != nil {
-	if t.config.isUnderlayServer == 1 {
+			return nil, err
-		t.pn.push(t.config.PeerNode, MsgPushUnderlayConnect, nil)
+		}
-		qConn, err = listenTCP(t.config.peerIP, t.config.peerConeNatPort, t.coneLocalPort, t.config.linkMode)
+	}
-		if err != nil {
+	conn, errL := net.ListenUDP("udp", listenAddr)
-			return nil, fmt.Errorf("listen TCP error:%s", err)
+	if errL != nil {
-		}
+		time.Sleep(time.Millisecond * 10)
-
+		conn, errL = net.ListenUDP("udp", listenAddr)
-		_, buff, err := qConn.ReadBuffer()
+		if errL != nil {
-		if err != nil {
+			return nil, fmt.Errorf("%s listen error:%s", underlayProtocol, errL)
-			return nil, fmt.Errorf("read start msg error:%s", err)
+		}
-		}
+	}
-		if buff != nil {
+	GNetwork.read(t.config.PeerNode, MsgPush, MsgPushUnderlayConnect, ReadMsgTimeout)
-			gLog.Println(LvDEBUG, string(buff))
+	gLog.d("%s dial to %s", underlayProtocol, t.remoteHoleAddr.String())
-		}
+	if t.config.UnderlayProtocol == "kcp" {
-		qConn.WriteBytes(MsgP2P, MsgTunnelHandshakeAck, []byte("OpenP2P,hello2"))
+		// ul, errL = dialKCP(conn, t.remoteHoleAddr, UnderlayConnectTimeout)
-		gLog.Println(LvINFO, "TCP connection ok")
+	} else {
-		return qConn, nil
+		ul, errL = dialQuic(conn, t.remoteHoleAddr, UnderlayConnectTimeout)
-	}
+	}
-
+
-	//else
+	if errL != nil {
-	t.pn.read(t.config.PeerNode, MsgPush, MsgPushUnderlayConnect, time.Second*5)
+		return nil, fmt.Errorf("%s dial to %s error:%s", underlayProtocol, t.remoteHoleAddr.String(), errL)
-	gLog.Println(LvDEBUG, "TCP dial to ", t.config.peerIP, ":", t.config.peerConeNatPort)
+	}
-	qConn, err = dialTCP(t.config.peerIP, t.config.peerConeNatPort, t.coneLocalPort, t.config.linkMode)
+	handshakeBegin := time.Now()
-	if err != nil {
+	tidBuff := new(bytes.Buffer)
-		return nil, fmt.Errorf("TCP dial to %s:%d error:%s", t.config.peerIP, t.config.peerConeNatPort, err)
+	binary.Write(tidBuff, binary.LittleEndian, t.id)
-	}
+	ul.WriteBytes(MsgP2P, MsgTunnelHandshake, tidBuff.Bytes())
-	handshakeBegin := time.Now()
+	_, buff, err := ul.ReadBuffer() // TODO: kcp need timeout
-	qConn.WriteBytes(MsgP2P, MsgTunnelHandshake, []byte("OpenP2P,hello"))
+	if err != nil {
-	_, buff, err := qConn.ReadBuffer()
+		ul.Close()
-	if err != nil {
+		return nil, fmt.Errorf("read MsgTunnelHandshake error:%s", err)
-		return nil, fmt.Errorf("read MsgTunnelHandshake error:%s", err)
+	}
-	}
+	if buff != nil {
-	if buff != nil {
+		gLog.d("handshake flag:%s", string(buff))
-		gLog.Println(LvDEBUG, string(buff))
+	}
-	}
+
-
+	gLog.i("rtt=%dms", time.Since(handshakeBegin)/time.Millisecond)
-	gLog.Println(LvINFO, "rtt=", time.Since(handshakeBegin))
+	gLog.i("%s connection ok", underlayProtocol)
-	gLog.Println(LvINFO, "TCP connection ok")
+	t.linkModeWeb = LinkModeUDPPunch
-	t.linkModeWeb = LinkModeIPv4
+	return ul, nil
-	return qConn, nil
+}
-}
+
-
+func (t *P2PTunnel) connectUnderlayTCP() (c underlay, err error) {
-func (t *P2PTunnel) connectUnderlayTCP6() (c underlay, err error) {
+	gLog.d("connectUnderlayTCP %s start ", t.config.LogPeerNode())
-	gLog.Println(LvINFO, "connectUnderlayTCP6 start")
+	defer gLog.d("connectUnderlayTCP %s end ", t.config.LogPeerNode())
-	defer gLog.Println(LvINFO, "connectUnderlayTCP6 end")
+	var ul underlay
-	var qConn *underlayTCP6
+	peerIP := t.config.peerIP
-	if t.config.isUnderlayServer == 1 {
+	if t.config.linkMode == LinkModeIntranet {
-		t.pn.push(t.config.PeerNode, MsgPushUnderlayConnect, nil)
+		peerIP = t.config.peerLanIP
-		qConn, err = listenTCP6(t.coneNatPort, TunnelIdleTimeout)
+	}
-		if err != nil {
+	// server side
-			return nil, fmt.Errorf("listen TCP6 error:%s", err)
+	if t.config.isUnderlayServer == 1 {
-		}
+		ul, err = listenTCP(peerIP, t.config.peerConeNatPort, t.coneLocalPort, t.config.linkMode, t)
-		_, buff, err := qConn.ReadBuffer()
+		if err != nil {
-		if err != nil {
+			return nil, fmt.Errorf("listen TCP error:%s", err)
-			qConn.listener.Close()
+		}
-			return nil, fmt.Errorf("read start msg error:%s", err)
+
-		}
+		t.linkModeWeb = LinkModeIPv4
-		if buff != nil {
+		if t.config.linkMode == LinkModeIntranet {
-			gLog.Println(LvDEBUG, string(buff))
+			t.linkModeWeb = LinkModeIntranet
-		}
+		}
-		qConn.WriteBytes(MsgP2P, MsgTunnelHandshakeAck, []byte("OpenP2P,hello2"))
+		if t.config.linkMode == LinkModeTCP6 {
-		gLog.Println(LvDEBUG, "TCP6 connection ok")
+			t.linkModeWeb = LinkModeIPv6
-		return qConn, nil
+		}
-	}
+		gLog.i("%s TCP connection ok", t.linkModeWeb)
-
+		return ul, nil
-	//else
+	}
-	t.pn.read(t.config.PeerNode, MsgPush, MsgPushUnderlayConnect, time.Second*5)
+
-	gLog.Println(LvDEBUG, "TCP6 dial to ", t.config.peerIPv6)
+	// client side
-	qConn, err = dialTCP6(t.config.peerIPv6, t.config.peerConeNatPort)
+	if t.config.linkMode == LinkModeTCP4 {
-	if err != nil {
+		GNetwork.read(t.config.PeerNode, MsgPush, MsgPushUnderlayConnect, ReadMsgTimeout)
-		return nil, fmt.Errorf("TCP6 dial to %s:%d error:%s", t.config.peerIPv6, t.config.peerConeNatPort, err)
+	} else { //tcp punch should sleep for punch the same time
-	}
+		if compareVersion(t.config.peerVersion, SyncServerTimeVersion) < 0 {
-	handshakeBegin := time.Now()
+			gLog.d("peer version %s less than %s", t.config.peerVersion, SyncServerTimeVersion)
-	qConn.WriteBytes(MsgP2P, MsgTunnelHandshake, []byte("OpenP2P,hello"))
+		} else {
-	_, buff, err := qConn.ReadBuffer()
+			ts := time.Duration(int64(t.punchTs) + GNetwork.dt + GNetwork.ddtma*int64(time.Since(GNetwork.hbTime)+PunchTsDelay)/int64(NetworkHeartbeatTime) - time.Now().UnixNano())
-	if err != nil {
+			if ts > PunchTsDelay || ts < 0 {
-		qConn.listener.Close()
+				ts = PunchTsDelay
-		return nil, fmt.Errorf("read MsgTunnelHandshake error:%s", err)
+			}
-	}
+			gLog.d("sleep %d ms", ts/time.Millisecond)
-	if buff != nil {
+			time.Sleep(ts)
-		gLog.Println(LvDEBUG, string(buff))
+		}
-	}
+	}
-
+	host := peerIP
-	gLog.Println(LvINFO, "rtt=", time.Since(handshakeBegin))
+	if t.config.linkMode == LinkModeTCP6 {
-	gLog.Println(LvDEBUG, "TCP6 connection ok")
+		host = t.config.peerIPv6
-	t.linkModeWeb = LinkModeIPv6
+	}
-	return qConn, nil
+	ul, err = dialTCP(host, t.config.peerConeNatPort, t.coneLocalPort, t.config.linkMode)
-}
+	if err != nil {
-
+		return nil, fmt.Errorf("TCP dial to %s:%d error:%s", host, t.config.peerConeNatPort, err)
-func (t *P2PTunnel) readLoop() {
+	}
-	decryptData := make([]byte, ReadBuffLen+PaddingSize) // 16 bytes for padding
+	handshakeBegin := time.Now()
-	gLog.Printf(LvDEBUG, "%d tunnel readloop start", t.id)
+	tidBuff := new(bytes.Buffer)
-	for t.isRuning() {
+	binary.Write(tidBuff, binary.LittleEndian, t.id)
-		t.conn.SetReadDeadline(time.Now().Add(TunnelIdleTimeout))
+	// fake_http_hostname := "speedtest.cn"
-		head, body, err := t.conn.ReadBuffer()
+	// user_agent := "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36"
-		if err != nil {
+	// ul.WriteMessage(MsgP2P, 100, fmt.Sprintf("GET / HTTP/1.1\r\nHost: %s\r\nUser-Agent: %s\r\nAccept: */*\r\n\r\n",
-			if t.isRuning() {
+	// 	fake_http_hostname, user_agent))
-				gLog.Printf(LvERROR, "%d tunnel read error:%s", t.id, err)
+	ul.WriteBytes(MsgP2P, MsgTunnelHandshake, tidBuff.Bytes()) //  tunnelID
-			}
+	_, buff, err := ul.ReadBuffer()
-			break
+	if err != nil {
-		}
+		return nil, fmt.Errorf("read MsgTunnelHandshake error:%s", err)
-		if head.MainType != MsgP2P {
+	}
-			continue
+	if buff != nil {
-		}
+		gLog.d("hello %s", string(buff))
-		switch head.SubType {
+	}
-		case MsgTunnelHeartbeat:
+
-			t.conn.WriteBytes(MsgP2P, MsgTunnelHeartbeatAck, nil)
+	gLog.i("rtt=%dms", time.Since(handshakeBegin)/time.Millisecond)
-			gLog.Printf(LvDEBUG, "%d read tunnel heartbeat", t.id)
+	t.linkModeWeb = LinkModeIPv4
-		case MsgTunnelHeartbeatAck:
+	if t.config.linkMode == LinkModeIntranet {
-			t.hbMtx.Lock()
+		t.linkModeWeb = LinkModeIntranet
-			t.hbTime = time.Now()
+	}
-			t.hbMtx.Unlock()
+	if t.config.linkMode == LinkModeTCP6 {
-			gLog.Printf(LvDEBUG, "%d read tunnel heartbeat ack", t.id)
+		t.linkModeWeb = LinkModeIPv6
-		case MsgOverlayData:
+	}
-			if len(body) < overlayHeaderSize {
+	gLog.i("%s TCP connection ok", t.linkModeWeb)
-				continue
+	return ul, nil
-			}
+}
-			overlayID := binary.LittleEndian.Uint64(body[:8])
+
-			gLog.Printf(LvDEBUG, "%d tunnel read overlay data %d", t.id, overlayID)
+func (t *P2PTunnel) connectUnderlayTCPSymmetric() (c underlay, err error) {
-			s, ok := t.overlayConns.Load(overlayID)
+	gLog.d("connectUnderlayTCPSymmetric %s start ", t.config.LogPeerNode())
-			if !ok {
+	defer gLog.d("connectUnderlayTCPSymmetric %s end ", t.config.LogPeerNode())
-				// debug level, when overlay connection closed, always has some packet not found tunnel
+	ts := time.Duration(int64(t.punchTs) + GNetwork.dt + GNetwork.ddtma*int64(time.Since(GNetwork.hbTime)+PunchTsDelay)/int64(NetworkHeartbeatTime) - time.Now().UnixNano())
-				gLog.Printf(LvDEBUG, "%d tunnel not found overlay connection %d", t.id, overlayID)
+	if ts > PunchTsDelay || ts < 0 {
-				continue
+		ts = PunchTsDelay
-			}
+	}
-			overlayConn, ok := s.(*overlayConn)
+	gLog.d("sleep %d ms", ts/time.Millisecond)
-			if !ok {
+	time.Sleep(ts)
-				continue
+	startTime := time.Now()
-			}
+	t.linkModeWeb = LinkModeTCPPunch
-			payload := body[overlayHeaderSize:]
+	gotCh := make(chan *underlayTCP, 1)
-			var err error
+	var wg sync.WaitGroup
-			if overlayConn.appKey != 0 {
+	var success atomic.Int32
-				payload, _ = decryptBytes(overlayConn.appKeyBytes, decryptData, body[overlayHeaderSize:], int(head.DataLen-uint32(overlayHeaderSize)))
+	if t.config.peerNatType == NATSymmetric { // c2s
-			}
+		randPorts := rand.Perm(65532)
-			_, err = overlayConn.Write(payload)
+		for i := 0; i < SymmetricHandshakeNum; i++ {
-			if err != nil {
+			wg.Add(1)
-				gLog.Println(LvERROR, "overlay write error:", err)
+			go func(port int) {
-			}
+				defer wg.Done()
-		case MsgRelayData:
+				ul, err := dialTCP(t.config.peerIP, port, t.coneLocalPort, LinkModeTCPPunch)
-			gLog.Printf(LvDEBUG, "got relay data datalen=%d", head.DataLen)
+				if err != nil {
-			if len(body) < 8 {
+					return
-				continue
+				}
-			}
+				if !success.CompareAndSwap(0, 1) {
-			tunnelID := binary.LittleEndian.Uint64(body[:8])
+					ul.Close() // only cone side close
-			t.pn.relay(tunnelID, body[8:])
+					return
-		case MsgRelayHeartbeat:
+				}
-			req := RelayHeartbeat{}
+				err = ul.WriteMessage(MsgP2P, MsgPunchHandshakeAck, P2PHandshakeReq{ID: t.id})
-			err := json.Unmarshal(body, &req)
+				if err != nil {
-			if err != nil {
+					ul.Close()
-				gLog.Printf(LvERROR, "wrong RelayHeartbeat:%s", err)
+					return
-				continue
+				}
-			}
+				_, buff, err := ul.ReadBuffer()
-			gLog.Printf(LvDEBUG, "got MsgRelayHeartbeat from %d:%d", req.RelayTunnelID, req.AppID)
+				if err != nil || buff == nil {
-			relayHead := new(bytes.Buffer)
+					gLog.d("c2s ul.ReadBuffer error:%s", err)
-			binary.Write(relayHead, binary.LittleEndian, req.RelayTunnelID)
+					return
-			msg, _ := newMessage(MsgP2P, MsgRelayHeartbeatAck, &req)
+				}
-			msgWithHead := append(relayHead.Bytes(), msg...)
+				req := P2PHandshakeReq{}
-			t.conn.WriteBytes(MsgP2P, MsgRelayData, msgWithHead)
+				if err = json.Unmarshal(buff, &req); err != nil {
-		case MsgRelayHeartbeatAck:
+					return
-			req := RelayHeartbeat{}
+				}
-			err := json.Unmarshal(body, &req)
+				if req.ID != t.id {
-			if err != nil {
+					return
-				gLog.Printf(LvERROR, "wrong RelayHeartbeat:%s", err)
+				}
-				continue
+				gLog.i("handshakeS2C TCP ok. cost %dms", time.Since(startTime)/time.Millisecond)
-			}
+
-			gLog.Printf(LvDEBUG, "got MsgRelayHeartbeatAck to %d", req.AppID)
+				gotCh <- ul
-			t.pn.updateAppHeartbeat(req.AppID)
+				close(gotCh)
-		case MsgOverlayConnectReq:
+			}(randPorts[i] + 2)
-			req := OverlayConnectReq{}
+		}
-			err := json.Unmarshal(body, &req)
+
-			if err != nil {
+	} else { // s2c
-				gLog.Printf(LvERROR, "wrong MsgOverlayConnectReq:%s", err)
+		for i := 0; i < SymmetricHandshakeNum; i++ {
-				continue
+			wg.Add(1)
-			}
+			go func() {
-			// app connect only accept token(not relay totp token), avoid someone using the share relay node's token
+				defer wg.Done()
-			if req.Token != t.pn.config.Token {
+				ul, err := dialTCP(t.config.peerIP, t.config.peerConeNatPort, 0, LinkModeTCPPunch)
-				gLog.Println(LvERROR, "Access Denied:", req.Token)
+				if err != nil {
-				continue
+					return
-			}
+				}
-
+
-			overlayID := req.ID
+				_, buff, err := ul.ReadBuffer()
-			gLog.Printf(LvDEBUG, "App:%d overlayID:%d connect %+v", req.AppID, overlayID, req)
+				if err != nil || buff == nil {
-			oConn := overlayConn{
+					gLog.d("s2c ul.ReadBuffer error:%s", err)
-				tunnel:   t,
+					return
-				id:       overlayID,
+				}
-				isClient: false,
+				req := P2PHandshakeReq{}
-				rtid:     req.RelayTunnelID,
+				if err = json.Unmarshal(buff, &req); err != nil {
-				appID:    req.AppID,
+					return
-				appKey:   GetKey(req.AppID),
+				}
-			}
+				if req.ID != t.id {
-			if req.Protocol == "udp" {
+					return
-				oConn.connUDP, err = net.DialUDP("udp", nil, &net.UDPAddr{IP: net.ParseIP(req.DstIP), Port: req.DstPort})
+				}
-			} else {
+				err = ul.WriteMessage(MsgP2P, MsgPunchHandshakeAck, P2PHandshakeReq{ID: t.id})
-				oConn.connTCP, err = net.DialTimeout("tcp", fmt.Sprintf("%s:%d", req.DstIP, req.DstPort), time.Second*5)
+				if err != nil {
-			}
+					ul.Close()
-			if err != nil {
+					return
-				gLog.Println(LvERROR, err)
+				}
-				continue
+				if success.CompareAndSwap(0, 1) {
-			}
+					gotCh <- ul
-
+					close(gotCh)
-			// calc key bytes for encrypt
+				}
-			if oConn.appKey != 0 {
+			}()
-				encryptKey := make([]byte, 16)
+		}
-				binary.LittleEndian.PutUint64(encryptKey, oConn.appKey)
+	}
-				binary.LittleEndian.PutUint64(encryptKey[8:], oConn.appKey)
+	select {
-				oConn.appKeyBytes = encryptKey
+	case <-time.After(HandshakeTimeout):
-			}
+		return nil, fmt.Errorf("wait tcp handshake timeout")
-
+	case ul := <-gotCh:
-			t.overlayConns.Store(oConn.id, &oConn)
+		return ul, nil
-			go oConn.run()
+	}
-		case MsgOverlayDisconnectReq:
+}
-			req := OverlayDisconnectReq{}
+
-			err := json.Unmarshal(body, &req)
+func (t *P2PTunnel) connectUnderlayTCP6() (c underlay, err error) {
-			if err != nil {
+	gLog.d("connectUnderlayTCP6 %s start ", t.config.LogPeerNode())
-				gLog.Printf(LvERROR, "wrong OverlayDisconnectRequest:%s", err)
+	defer gLog.d("connectUnderlayTCP6 %s end ", t.config.LogPeerNode())
-				continue
+	tidBuff := new(bytes.Buffer)
-			}
+	binary.Write(tidBuff, binary.LittleEndian, t.id)
-			overlayID := req.ID
+	if t.config.isUnderlayServer == 1 {
-			gLog.Printf(LvDEBUG, "%d disconnect overlay connection %d", t.id, overlayID)
+		GNetwork.push(t.config.PeerNode, MsgPushUnderlayConnect, nil)
-			i, ok := t.overlayConns.Load(overlayID)
+		// ul, err = listenTCP6(t.coneNatPort, UnderlayConnectTimeout)
-			if ok {
+		tid := t.id
-				oConn := i.(*overlayConn)
+		if compareVersion(t.config.peerVersion, PublicIPVersion) < 0 { // old version
-				oConn.running = false
+			ipBytes := net.ParseIP(t.config.peerIP).To4()
-			}
+			tid = uint64(binary.BigEndian.Uint32(ipBytes))
-		default:
+			gLog.d("compatible with old client, use ip as key:%d", tid)
-		}
+		}
-	}
+
-	t.setRun(false)
+		if v4l != nil {
-	t.conn.Close()
+			c = v4l.getUnderlay(tid)
-	gLog.Printf(LvDEBUG, "%d tunnel readloop end", t.id)
+		}
-}
+		if c == nil {
-
+			return nil, fmt.Errorf("listen TCP6 error:%s", err)
-func (t *P2PTunnel) heartbeatLoop() {
+		}
-	tc := time.NewTicker(TunnelHeartbeatTime)
+		_, buff, err := c.ReadBuffer()
-	defer tc.Stop()
+		if err != nil {
-	gLog.Printf(LvDEBUG, "%d tunnel heartbeatLoop start", t.id)
+			return nil, fmt.Errorf("read start msg error:%s", err)
-	defer gLog.Printf(LvDEBUG, "%d tunnel heartbeatLoop end", t.id)
+		}
-	for t.isRuning() {
+		if buff != nil {
-		select {
+			gLog.d("handshake flag:%s", string(buff))
-		case <-tc.C:
+		}
-			// tunnel send
+		c.WriteBytes(MsgP2P, MsgTunnelHandshake, tidBuff.Bytes()) //  tunnelID
-			err := t.conn.WriteBytes(MsgP2P, MsgTunnelHeartbeat, nil)
+		// ul.WriteBytes(MsgP2P, MsgTunnelHandshakeAck, []byte("OpenP2P,hello2"))
-			if err != nil {
+		gLog.d("TCP6 connection ok")
-				gLog.Printf(LvERROR, "%d write tunnel heartbeat error %s", t.id, err)
+		t.linkModeWeb = LinkModeIPv6
-				t.setRun(false)
+		return c, nil
-				return
+	}
-			}
+
-			gLog.Printf(LvDEBUG, "%d write tunnel heartbeat ok", t.id)
+	//else
-		}
+	GNetwork.read(t.config.PeerNode, MsgPush, MsgPushUnderlayConnect, ReadMsgTimeout)
-	}
+	gLog.d("TCP6 dial to %s", t.config.peerIPv6)
-}
+	ul, err := dialTCP(fmt.Sprintf("[%s]", t.config.peerIPv6), t.config.peerConeNatPort, 0, LinkModeTCP6)
-
+	if err != nil || ul == nil {
-func (t *P2PTunnel) listen() error {
+		return nil, fmt.Errorf("TCP6 dial to %s:%d error:%s", t.config.peerIPv6, t.config.peerConeNatPort, err)
-	// notify client to connect
+	}
-	rsp := PushConnectRsp{
+	handshakeBegin := time.Now()
-		Error:   0,
+	ul.WriteBytes(MsgP2P, MsgTunnelHandshake, tidBuff.Bytes()) //  tunnelID
-		Detail:  "connect ok",
+	// ul.WriteBytes(MsgP2P, MsgTunnelHandshake, []byte("OpenP2P,hello"))
-		To:      t.config.PeerNode,
+	_, buff, errR := ul.ReadBuffer()
-		From:    t.pn.config.Node,
+	if errR != nil {
-		NatType: t.pn.config.natType,
+		return nil, fmt.Errorf("read MsgTunnelHandshake error:%s", errR)
-		HasIPv4: t.pn.config.hasIPv4,
+	}
-		// IPv6:            t.pn.config.IPv6,
+	if buff != nil {
-		HasUPNPorNATPMP: t.pn.config.hasUPNPorNATPMP,
+		gLog.d("handshake flag:%s", string(buff))
-		FromIP:          t.pn.config.publicIP,
+	}
-		ConeNatPort:     t.coneNatPort,
+
-		ID:              t.id,
+	gLog.i("rtt=%dms", time.Since(handshakeBegin))
-		Version:         OpenP2PVersion,
+	gLog.i("TCP6 connection ok")
-	}
+	t.linkModeWeb = LinkModeIPv6
-	// only private node set ipv6
+	return ul, nil
-	if t.config.fromToken == t.pn.config.Token {
+}
-		t.pn.refreshIPv6(false)
+
-		rsp.IPv6 = t.pn.config.publicIPv6
+func (t *P2PTunnel) readLoop() {
-	}
+	decryptData := make([]byte, ReadBuffLen+PaddingSize) // 16 bytes for padding
-
+	gLog.d("%d tunnel readloop start", t.id)
-	t.pn.push(t.config.PeerNode, MsgPushConnectRsp, rsp)
+	for t.isRuning() {
-	gLog.Printf(LvDEBUG, "p2ptunnel wait for connecting")
+		t.conn.SetReadDeadline(time.Now().Add(TunnelHeartbeatTime * 2))
-	t.tunnelServer = true
+		head, body, err := t.conn.ReadBuffer()
-	return t.start()
+		if err != nil || head == nil {
-}
+			if t.isRuning() {
-
+				gLog.d("%d tunnel read error:%s", t.id, err)
-func (t *P2PTunnel) closeOverlayConns(appID uint64) {
+			}
-	t.overlayConns.Range(func(_, i interface{}) bool {
+			break
-		oConn := i.(*overlayConn)
+		}
-		if oConn.appID == appID {
+		if head.MainType != MsgP2P {
-			if oConn.connTCP != nil {
+			gLog.w("%d head.MainType(%d) != MsgP2P", head.MainType, t.id)
-				oConn.connTCP.Close()
+			continue
-				oConn.connTCP = nil
+		}
-			}
+		// gLog.d("%d tunnel read %d:%d len=%d", t.id, head.MainType, head.SubType, head.DataLen)
-			if oConn.connUDP != nil {
+		// TODO: replace some case implement to functions
-				oConn.connUDP.Close()
+		switch head.SubType {
-				oConn.connUDP = nil
+		case MsgTunnelHeartbeat:
-			}
+			t.hbMtx.Lock()
-		}
+			t.hbTime = time.Now()
-		return true
+			t.hbMtx.Unlock()
-	})
+			memAppPeerID := new(bytes.Buffer)
-}
+			binary.Write(memAppPeerID, binary.LittleEndian, gConf.Network.nodeID)
+			t.conn.WriteBytes(MsgP2P, MsgTunnelHeartbeatAck, memAppPeerID.Bytes())
+			gLog.dev("%d read tunnel heartbeat", t.id)
+		case MsgTunnelHeartbeatAck:
+			t.hbMtx.Lock()
+			t.hbTime = time.Now()
+			t.hbMtx.Unlock()
+			if head.DataLen >= 8 {
+				memAppPeerID := binary.LittleEndian.Uint64(body[:8])
+				existApp, appok := GNetwork.apps.Load(memAppPeerID)
+				if appok {
+					app := existApp.(*p2pApp)
+					for i := 0; i < app.relayIdxStart; i++ {
+						if app.Tunnel(i) == t {
+							app.rtt[i].Store(int32(time.Since(t.whbTime) / time.Millisecond))
+							break
+						}
+					}
+				}
+			}
+
+			gLog.dev("%d read tunnel heartbeat ack, rtt=%dms", t.id, time.Since(t.whbTime)/time.Millisecond)
+		case MsgOverlayData:
+			if len(body) < overlayHeaderSize {
+				gLog.w("%d len(body) < overlayHeaderSize", t.id)
+				continue
+			}
+			overlayID := binary.LittleEndian.Uint64(body[:8])
+			gLog.dev("%d tunnel read overlay data %d bodylen=%d", t.id, overlayID, head.DataLen)
+			s, ok := overlayConns.Load(overlayID)
+			if !ok {
+				// debug level, when overlay connection closed, always has some packet not found tunnel
+				gLog.d("%d tunnel not found overlay connection %d", t.id, overlayID)
+				continue
+			}
+			overlayConn, ok := s.(*overlayConn)
+			if !ok {
+				continue
+			}
+			payload := body[overlayHeaderSize:]
+			var err error
+			if overlayConn.app.key != 0 {
+				payload, _ = decryptBytes(overlayConn.app.appKeyBytes, decryptData, body[overlayHeaderSize:], int(head.DataLen-uint32(overlayHeaderSize)))
+			}
+			_, err = overlayConn.Write(payload)
+			if err != nil {
+				gLog.e("overlay write error:%s", err)
+			}
+		case MsgNodeDataMP:
+			t.handleNodeDataMP(head, body)
+		case MsgNodeDataMPAck:
+			t.handleNodeDataMPAck(head, body)
+		case MsgNodeData: // unused
+			t.handleNodeData(head, body, false)
+		case MsgRelayNodeData: // unused
+			t.handleNodeData(head, body, true)
+		case MsgRelayData:
+			if len(body) < 8 {
+				continue
+			}
+			tunnelID := binary.LittleEndian.Uint64(body[:8])
+			gLog.dev("relay data to %d, len=%d", tunnelID, head.DataLen-RelayHeaderSize)
+			if err := GNetwork.relay(tunnelID, body[RelayHeaderSize:]); err != nil {
+				gLog.d("%s:%d relay to %d len=%d error:%s", t.config.LogPeerNode(), t.id, tunnelID, len(body), ErrRelayTunnelNotFound)
+			}
+		case MsgRelayHeartbeat: // only client side will write relay heartbeat, different with tunnel heartbeat
+			req := RelayHeartbeat{}
+			if err := json.Unmarshal(body, &req); err != nil {
+				gLog.e("wrong %v:%s", reflect.TypeOf(req), err)
+				continue
+			}
+			// TODO: debug relay heartbeat
+			gLog.dev("read MsgRelayHeartbeat from rtid:%d,appid:%d", req.RelayTunnelID, req.AppID)
+			// update app hbtime
+			GNetwork.updateAppHeartbeat(req.AppID, req.RelayTunnelID, true)
+			req.From = gConf.Network.Node
+			t.WriteMessage(req.RelayTunnelID, MsgP2P, MsgRelayHeartbeatAck, &req)
+		case MsgRelayHeartbeatAck:
+			req := RelayHeartbeat{}
+			err := json.Unmarshal(body, &req)
+			if err != nil {
+				gLog.e("wrong RelayHeartbeat:%s", err)
+				continue
+			}
+			// TODO: debug relay heartbeat
+			gLog.dev("read MsgRelayHeartbeatAck to appid:%d", req.AppID)
+			GNetwork.updateAppHeartbeat(req.AppID, req.RelayTunnelID, false)
+			req.From = gConf.Network.Node
+			t.WriteMessage(req.RelayTunnelID2, MsgP2P, MsgRelayHeartbeatAck2, &req)
+		case MsgRelayHeartbeatAck2:
+			req := RelayHeartbeat{}
+			err := json.Unmarshal(body, &req)
+			if err != nil {
+				gLog.e("wrong RelayHeartbeat:%s", err)
+				continue
+			}
+			gLog.dev("read MsgRelayHeartbeatAck2 to appid:%d", req.AppID)
+			GNetwork.updateAppHeartbeat(req.AppID, req.RelayTunnelID, false)
+		case MsgOverlayConnectReq: // TODO: send this msg withAppID, and app handle it
+			// app connect only accept token(not relay totp token), avoid someone using the share relay node's token
+			// targetApp := GNetwork.GetAPPByID(req.AppID)
+			t.handleOverlayConnectReq(body, err)
+		case MsgOverlayConnectRsp:
+			appID := binary.LittleEndian.Uint64(body[:8])
+			i, ok := GNetwork.apps.Load(appID)
+			if !ok {
+				gLog.e("MsgOverlayConnectRsp app not found %d", appID)
+				return
+			}
+			app := i.(*p2pApp)
+			// ndmp := NodeDataMPHeader{fromNodeID: gConf.Network.nodeID, seq: seq}
+			app.StoreMessage(head, body)
+		case MsgOverlayDisconnectReq:
+			req := OverlayDisconnectReq{}
+			if err := json.Unmarshal(body, &req); err != nil {
+				gLog.e("wrong %v:%s", reflect.TypeOf(req), err)
+				continue
+			}
+			overlayID := req.ID
+			gLog.d("%d disconnect overlay connection %d", t.id, overlayID)
+			i, ok := overlayConns.Load(overlayID)
+			if ok {
+				oConn := i.(*overlayConn)
+				oConn.Close()
+			}
+		default:
+		}
+	}
+	t.close()
+	gLog.d("%d tunnel readloop end", t.id)
+}
+
+func (*P2PTunnel) handleOverlayConnectReq(body []byte, err error) {
+	req := OverlayConnectReq{}
+	if err := json.Unmarshal(body, &req); err != nil {
+		gLog.e("wrong %v:%s", reflect.TypeOf(req), err)
+		return
+	}
+
+	if req.Token != gConf.Network.Token {
+		gLog.e("Access Denied,token=%d", req.Token)
+		return
+	}
+
+	overlayID := req.ID
+	gLog.d("App:%d overlayID:%d connect %s:%d", req.AppID, overlayID, req.DstIP, req.DstPort)
+
+	i, ok := GNetwork.apps.Load(req.AppID)
+	if !ok {
+		return
+	}
+	targetApp := i.(*p2pApp)
+	oConn := overlayConn{
+		app:      targetApp,
+		id:       overlayID,
+		isClient: false,
+		running:  true,
+	}
+	// connect local service should use sys dns
+	sysResolver := &net.Resolver{}
+	ips, err := sysResolver.LookupIP(context.Background(), "ip4", req.DstIP)
+	if err != nil {
+		gLog.e("handleOverlayConnectReq dial error:%s", err)
+		return
+	}
+	if req.Protocol == "udp" {
+		oConn.connUDP, err = net.DialUDP("udp", nil, &net.UDPAddr{IP: ips[0], Port: req.DstPort})
+	} else {
+		oConn.connTCP, err = net.DialTimeout("tcp", fmt.Sprintf("%s:%d", ips[0].String(), req.DstPort), ReadMsgTimeout)
+
+	}
+	if err != nil {
+		gLog.e("handleOverlayConnectReq dial error:%s", err)
+		return
+	}
+	overlayConns.Store(oConn.id, &oConn)
+	go oConn.run()
+	targetApp.WriteMessageWithAppID(MsgP2P, MsgOverlayConnectRsp, nil)
+}
+
+func (t *P2PTunnel) writeLoop() {
+	t.hbMtx.Lock()
+	t.hbTime = time.Now() // init
+	t.hbMtx.Unlock()
+	tc := time.NewTicker(TunnelHeartbeatTime)
+	defer tc.Stop()
+	gLog.d("%s:%d tunnel writeLoop start", t.config.LogPeerNode(), t.id)
+	defer gLog.d("%s:%d tunnel writeLoop end", t.config.LogPeerNode(), t.id)
+	writeHb := func() {
+		// tunnel send
+		t.whbTime = time.Now()
+		err := t.conn.WriteBytes(MsgP2P, MsgTunnelHeartbeat, nil)
+		if err != nil {
+			gLog.d("%d write tunnel heartbeat error %s", t.id, err)
+			t.close()
+			return
+		}
+		gLog.dev("%d write tunnel heartbeat ok", t.id)
+	}
+	writeHb()
+	for t.isRuning() {
+		select {
+		case buff := <-t.writeDataSmall:
+			t.conn.WriteBuffer(buff)
+			// gLog.d("write icmp %d", time.Now().Unix())
+		default:
+			select {
+			case buff := <-t.writeDataSmall:
+				t.conn.WriteBuffer(buff)
+				// gLog.d("write icmp %d", time.Now().Unix())
+			case buff := <-t.writeData:
+				err := t.conn.WriteBuffer(buff)
+				if err != nil {
+					gLog.e("%d write tunnel error %s", t.id, err)
+					t.close()
+					return
+				}
+			case <-tc.C:
+				writeHb()
+			}
+		}
+	}
+}
+
+func (t *P2PTunnel) listen() error {
+	// notify client to connect
+	rsp := PushConnectRsp{
+		Error:   0,
+		Detail:  "connect ok",
+		To:      t.config.PeerNode,
+		From:    gConf.Network.Node,
+		NatType: gConf.Network.natType,
+		HasIPv4: gConf.Network.hasIPv4,
+		// IPv6:            gConf.Network.IPv6,
+		HasUPNPorNATPMP: gConf.Network.hasUPNPorNATPMP,
+		FromIP:          gConf.Network.publicIP,
+		ConeNatPort:     t.coneNatPort,
+		ID:              t.id,
+		PunchTs:         uint64(time.Now().UnixNano() + int64(PunchTsDelay) - GNetwork.dt),
+		Version:         OpenP2PVersion,
+	}
+	t.punchTs = rsp.PunchTs
+	// only private node set ipv6
+	if t.config.fromToken == gConf.Network.Token {
+		rsp.IPv6 = gConf.IPv6()
+	}
+
+	GNetwork.push(t.config.PeerNode, MsgPushConnectRsp, rsp)
+	gLog.d("p2ptunnel wait for connecting")
+	return t.start()
+}
+
+func (t *P2PTunnel) handleNodeData(head *openP2PHeader, body []byte, isRelay bool) {
+	gLog.dev("%d tunnel read node data bodylen=%d, relay=%t", t.id, head.DataLen, isRelay)
+	ch := GNetwork.nodeData
+	// if body[9] == 1 { // TODO: deal relay
+	// 	ch = GNetwork.nodeDataSmall
+	// 	gLog.d("read icmp %d", time.Now().Unix())
+	// }
+	if isRelay {
+		// fromPeerID := binary.LittleEndian.Uint64(body[:8]) // unused
+		ch <- body[8:] // TODO: cache peerNodeID; encrypt/decrypt
+	} else {
+		ch <- body // TODO: cache peerNodeID; encrypt/decrypt
+	}
+}
+
+func (t *P2PTunnel) handleNodeDataMP(head *openP2PHeader, body []byte) {
+	gLog.dev("%s tid:%d tunnel read node data mp bodylen=%d", t.config.LogPeerNode(), t.id, head.DataLen) // Debug
+	if head.DataLen < 16 {
+		return
+	}
+
+	// TODO: reorder write tun
+	fromNodeID := binary.LittleEndian.Uint64(body[:8])
+	seq := binary.LittleEndian.Uint64(body[8:16])
+	i, ok := GNetwork.apps.Load(fromNodeID)
+	if !ok {
+		gLog.e("handleNodeDataMP peer not found,from=%s nodeID=%d, seq=%d", t.config.LogPeerNode(), fromNodeID, seq)
+		return
+	}
+	app := i.(*p2pApp)
+	// ndmp := NodeDataMPHeader{fromNodeID: gConf.Network.nodeID, seq: seq}
+	app.handleNodeDataMP(seq, body[16:], t)
+
+}
+func (t *P2PTunnel) handleNodeDataMPAck(head *openP2PHeader, body []byte) {
+
+}
+
+func (t *P2PTunnel) asyncWriteNodeData(id uint64, seq uint64, IPPacket []byte, relayHead []byte) {
+	all := new(bytes.Buffer)
+	if relayHead != nil {
+		all.Write(encodeHeader(MsgP2P, MsgRelayData, uint32(openP2PHeaderSize+len(relayHead)+16+len(IPPacket))))
+		all.Write(relayHead)
+	}
+	all.Write(encodeHeader(MsgP2P, MsgNodeDataMP, 16+uint32(len(IPPacket)))) // id+seq=16 bytes
+	binary.Write(all, binary.LittleEndian, id)
+	binary.Write(all, binary.LittleEndian, seq)
+	all.Write(IPPacket)
+	// if len(data) < 192 {
+	if IPPacket[9] == 1 { // icmp
+		select {
+		case t.writeDataSmall <- all.Bytes():
+			// gLog.w("%s:%d t.writeDataSmall write %d", t.config.PeerNode, t.id, len(t.writeDataSmall))
+		default:
+			gLog.w("%s:%d t.writeDataSmall is full, drop it", t.config.LogPeerNode(), t.id)
+		}
+	} else {
+		t.writeData <- all.Bytes()
+		// select {
+		// case t.writeData <- writeBytes:
+		// default:
+		// 	gLog.w("%s:%d t.writeData is full, drop it", t.config.LogPeerNode(), t.id)
+		// }
+	}
+}
+
+func (t *P2PTunnel) WriteMessage(rtid uint64, mainType uint16, subType uint16, req interface{}) error {
+	if rtid == 0 {
+		return t.conn.WriteMessage(mainType, subType, &req)
+	}
+	relayHead := new(bytes.Buffer)
+	binary.Write(relayHead, binary.LittleEndian, rtid)
+	msg, _ := newMessage(mainType, subType, &req)
+	msgWithHead := append(relayHead.Bytes(), msg...)
+	return t.conn.WriteBytes(mainType, MsgRelayData, msgWithHead)
+
+}
+
+func (t *P2PTunnel) WriteMessageWithAppID(appID uint64, rtid uint64, mainType uint16, subType uint16, req interface{}) error {
+	data, err := json.Marshal(req)
+	if err != nil {
+		return err
+	}
+	head := new(bytes.Buffer)
+	binary.Write(head, binary.LittleEndian, appID)
+	msgWithAppID := append(head.Bytes(), data...)
+	if rtid == 0 {
+		return t.conn.WriteBytes(mainType, subType, msgWithAppID)
+	}
+	relayHead := new(bytes.Buffer)
+	binary.Write(relayHead, binary.LittleEndian, rtid)
+	msg, _ := newMessageWithBuff(mainType, subType, msgWithAppID)
+	msgWithHead := append(relayHead.Bytes(), msg...)
+	return t.conn.WriteBytes(mainType, MsgRelayData, msgWithHead)
+
+}
+
+func (t *P2PTunnel) WriteBytes(rtid uint64, mainType uint16, subType uint16, data []byte) error {
+	if rtid == 0 {
+		return t.conn.WriteBytes(mainType, subType, data)
+	}
+	all := new(bytes.Buffer)
+	binary.Write(all, binary.LittleEndian, rtid)
+	all.Write(encodeHeader(mainType, subType, uint32(len(data))))
+	all.Write(data)
+	return t.conn.WriteBytes(mainType, MsgRelayData, all.Bytes())
+
+}
+
+// func (t *P2PTunnel) RTT() int {
+// 	if t.isWriting.Load() && t.rtt.Load() < int32(time.Now().Add(time.Duration(-t.writingTs.Load())).Unix()/int64(time.Millisecond)) {
+// 		return int(time.Now().Add(time.Duration(-t.writingTs.Load())).Unix() / int64(time.Millisecond))
+// 	}
+// 	return int(t.rtt.Load())
+// }

core/p2ptunnel_test.go

@@ -0,0 +1,24 @@
+package openp2p
+
+import (
+	"fmt"
+	"testing"
+)
+
+func TestSelectPriority(t *testing.T) {
+	writeData := make(chan []byte, WriteDataChanSize)
+	writeDataSmall := make(chan []byte, WriteDataChanSize/30)
+	for i := 0; i < 100; i++ {
+		writeData <- []byte("data")
+		writeDataSmall <- []byte("small data")
+	}
+	for i := 0; i < 100; i++ {
+		select {
+		case buff := <-writeDataSmall:
+			fmt.Printf("got small data:%s\n", string(buff))
+		case buff := <-writeData:
+			fmt.Printf("got data:%s\n", string(buff))
+		}
+	}
+
+}

core/ping.go

@@ -0,0 +1,87 @@
+package openp2p
+
+import (
+	"fmt"
+	"net"
+	"os"
+	"time"
+
+	"golang.org/x/net/icmp"
+	"golang.org/x/net/ipv4"
+)
+
+// 定义ICMP回显请求和应答的结构
+type ICMPMessage struct {
+	Type     uint8
+	Code     uint8
+	Checksum uint16
+	Ident    uint16
+	Seq      uint16
+	Data     []byte
+}
+
+// Ping sends an ICMP Echo request to the specified host and returns the response time.
+func Ping(host string) (time.Duration, error) {
+	// Resolve the IP address of the host
+	ipAddr, err := net.ResolveIPAddr("ip4", host)
+	if err != nil {
+		return 0, fmt.Errorf("failed to resolve host: %v", err)
+	}
+
+	// Create an ICMP listener
+	conn, err := net.ListenPacket("ip4:icmp", "0.0.0.0")
+	if err != nil {
+		return 0, fmt.Errorf("failed to create ICMP connection: %v", err)
+	}
+	defer conn.Close()
+
+	// Create an ICMP Echo request message
+	message := icmp.Message{
+		Type: ipv4.ICMPTypeEcho,
+		Code: 0,
+		Body: &icmp.Echo{
+			ID:   os.Getpid() & 0xffff,
+			Seq:  1,
+			Data: []byte("HELLO-R-U-THERE"),
+		},
+	}
+
+	// Marshal the message into binary form
+	messageBytes, err := message.Marshal(nil)
+	if err != nil {
+		return 0, fmt.Errorf("failed to marshal ICMP message: %v", err)
+	}
+
+	// Send the ICMP Echo request
+	start := time.Now()
+	if _, err := conn.WriteTo(messageBytes, ipAddr); err != nil {
+		return 0, fmt.Errorf("failed to send ICMP request: %v", err)
+	}
+
+	// Set a deadline for the response
+	err = conn.SetReadDeadline(time.Now().Add(3 * time.Second))
+	if err != nil {
+		return 0, fmt.Errorf("failed to set read deadline: %v", err)
+	}
+
+	// Read the ICMP response
+	response := make([]byte, 1500)
+	n, _, err := conn.ReadFrom(response)
+	if err != nil {
+		return 0, fmt.Errorf("failed to read ICMP response: %v", err)
+	}
+
+	// Parse the ICMP response message
+	parsedMessage, err := icmp.ParseMessage(ipv4.ICMPTypeEchoReply.Protocol(), response[:n])
+	if err != nil {
+		return 0, fmt.Errorf("failed to parse ICMP response: %v", err)
+	}
+
+	// Check if the response is an Echo reply
+	if parsedMessage.Type == ipv4.ICMPTypeEchoReply {
+		duration := time.Since(start)
+		return duration, nil
+	} else {
+		return 0, fmt.Errorf("unexpected ICMP message: %+v", parsedMessage)
+	}
+}

core/protocol.go

@@ -10,16 +10,24 @@ import (
 	"time"
 )
 
-const OpenP2PVersion = "3.5.2"
+const OpenP2PVersion = "3.25.11"
-const ProducnName string = "openp2p"
+const ProductName string = "openp2p"
 const LeastSupportVersion = "3.0.0"
-
+const SyncServerTimeVersion = "3.9.0"
+const SymmetricSimultaneouslySendVersion = "3.10.7"
+const PublicIPVersion = "3.11.2"
+const SupportIntranetVersion = "3.14.5"
+const SupportDualTunnelVersion = "3.15.5"
+const IPv6PunchVersion = "3.24.9"
+const SupportUDP4DirectVersion = "3.24.16"
+const SupportMultiDirectVersion = "3.25.1"
 const (
-	IfconfigPort1 = 27180
+	NATDetectPort1 = 27180
-	IfconfigPort2 = 27181
+	NATDetectPort2 = 27181
-	WsPort        = 27183
+	WsPort         = 27183
-	UDPPort1      = 27182
+	WsPort2        = 465
-	UDPPort2      = 27183
+	UDPPort1       = 27182
+	UDPPort2       = 27183
 )
 
 type openP2PHeader struct {
@@ -37,10 +45,18 @@ type PushHeader struct {
 
 var PushHeaderSize = binary.Size(PushHeader{})
 
+const RelayHeaderSize = 8
+
 type overlayHeader struct {
 	id uint64
 }
 
+type NodeDataMPAck struct {
+	FromNodeID uint64
+	Seq        uint64
+	Delay      uint32 // delay write mergeack ms
+}
+
 var overlayHeaderSize = binary.Size(overlayHeader{})
 
 func decodeHeader(data []byte) (*openP2PHeader, error) {
@@ -67,7 +83,7 @@ func encodeHeader(mainType uint16, subType uint16, len uint32) []byte {
 	return headBuf.Bytes()
 }
 
-// Message type
+// Message main type
 const (
 	MsgLogin     = 0
 	MsgHeartbeat = 1
@@ -77,78 +93,103 @@ const (
 	MsgRelay     = 5
 	MsgReport    = 6
 	MsgQuery     = 7
+	MsgSDWAN     = 8
 )
 
 // TODO: seperate node push and web push.
 const (
-	MsgPushRsp               = 0
+	MsgPushRsp                  = 0
-	MsgPushConnectReq        = 1
+	MsgPushConnectReq           = 1
-	MsgPushConnectRsp        = 2
+	MsgPushConnectRsp           = 2
-	MsgPushHandshakeStart    = 3
+	MsgPushHandshakeStart       = 3
-	MsgPushAddRelayTunnelReq = 4
+	MsgPushAddRelayTunnelReq    = 4
-	MsgPushAddRelayTunnelRsp = 5
+	MsgPushAddRelayTunnelRsp    = 5
-	MsgPushUpdate            = 6
+	MsgPushUpdate               = 6
-	MsgPushReportApps        = 7
+	MsgPushReportApps           = 7
-	MsgPushUnderlayConnect   = 8
+	MsgPushUnderlayConnect      = 8
-	MsgPushEditApp           = 9
+	MsgPushEditApp              = 9
-	MsgPushSwitchApp         = 10
+	MsgPushSwitchApp            = 10
-	MsgPushRestart           = 11
+	MsgPushRestart              = 11
-	MsgPushEditNode          = 12
+	MsgPushEditNode             = 12
-	MsgPushAPPKey            = 13
+	MsgPushAPPKey               = 13
-	MsgPushReportLog         = 14
+	MsgPushReportLog            = 14
+	MsgPushDstNodeOnline        = 15
+	MsgPushReportGoroutine      = 16
+	MsgPushReportMemApps        = 17
+	MsgPushServerSideSaveMemApp = 18
+	MsgPushCheckRemoteService   = 19
+	MsgPushSpecTunnel           = 20
+	MsgPushReportHeap           = 21
+	MsgPushSDWanRefresh         = 22
+	MsgPushNat4Detect           = 23
 )
 
 // MsgP2P sub type message
 const (
-	MsgPunchHandshake = iota
+	MsgPunchHandshake       = 0
-	MsgPunchHandshakeAck
+	MsgPunchHandshakeAck    = 1
-	MsgTunnelHandshake
+	MsgTunnelHandshake      = 2
-	MsgTunnelHandshakeAck
+	MsgTunnelHandshakeAck   = 3
-	MsgTunnelHeartbeat
+	MsgTunnelHeartbeat      = 4
-	MsgTunnelHeartbeatAck
+	MsgTunnelHeartbeatAck   = 5
-	MsgOverlayConnectReq
+	MsgOverlayConnectReq    = 6
-	MsgOverlayConnectRsp
+	MsgOverlayConnectRsp    = 7
-	MsgOverlayDisconnectReq
+	MsgOverlayDisconnectReq = 8
-	MsgOverlayData
+	MsgOverlayData          = 9
-	MsgRelayData
+	MsgRelayData            = 10
-	MsgRelayHeartbeat
+	MsgRelayHeartbeat       = 11
-	MsgRelayHeartbeatAck
+	MsgRelayHeartbeatAck    = 12
+	MsgNodeData             = 13
+	MsgRelayNodeData        = 14
+	MsgNodeDataMP           = 15
+	MsgNodeDataMPAck        = 16
+	MsgRelayHeartbeatAck2   = 17
 )
 
 // MsgRelay sub type message
 const (
-	MsgRelayNodeReq = iota
+	MsgRelayNodeReq = 0
-	MsgRelayNodeRsp
+	MsgRelayNodeRsp = 1
 )
 
 // MsgReport sub type message
 const (
-	MsgReportBasic = iota
+	MsgReportBasic    = 0
-	MsgReportQuery
+	MsgReportQuery    = 1
-	MsgReportConnect
+	MsgReportConnect  = 2
-	MsgReportApps
+	MsgReportApps     = 3
-	MsgReportLog
+	MsgReportLog      = 4
+	MsgReportMemApps  = 5
+	MsgReportResponse = 6
+	MsgReportBasicRsp = 7
 )
 
 const (
-	ReadBuffLen           = 4096             // for UDP maybe not enough
+	ReadBuffLen               = 4096 // for UDP maybe not enough
-	NetworkHeartbeatTime  = time.Second * 30 // TODO: server no response hb, save flow
+	NetworkHeartbeatTime      = time.Second * 30
-	TunnelHeartbeatTime   = time.Second * 15
+	TunnelHeartbeatTime       = time.Second * 10 // some nat udp session expired time less than 15s. change to 10s
-	TunnelIdleTimeout     = time.Minute
+	UnderlayTCPKeepalive      = time.Second * 5
-	SymmetricHandshakeNum = 800 // 0.992379
+	UnderlayTCPConnectTimeout = time.Second * 5
+	TunnelIdleTimeout         = time.Minute
+	SymmetricHandshakeNum     = 800 // 0.992379
 	// SymmetricHandshakeNum        = 1000 // 0.999510
-	SymmetricHandshakeInterval   = time.Millisecond
+	SymmetricHandshakeInterval = time.Millisecond
-	SymmetricHandshakeAckTimeout = time.Second * 11
+	HandshakeTimeout           = time.Second * 7
-	PeerAddRelayTimeount         = time.Second * 20
+	PunchTsDelay               = time.Second * 3
-	CheckActiveTimeout           = time.Second * 5
+	PeerAddRelayTimeount       = time.Second * 30 // peer need times. S2C\TCP\TCP Punch\UDP Punch
-	PaddingSize                  = 16
+	CheckActiveTimeout         = time.Second * 5
-	AESKeySize                   = 16
+	ReadMsgTimeout             = time.Second * 5
-	MaxRetry                     = 10
+	PaddingSize                = 16
-	RetryInterval                = time.Second * 30
+	AESKeySize                 = 16
-	PublicIPEchoTimeout          = time.Second * 1
+	MaxRetry                   = 10
-	NatTestTimeout               = time.Second * 10
+	Cone2ConeTCPPunchMaxRetry  = 1
-	ClientAPITimeout             = time.Second * 10
+	Cone2ConeUDPPunchMaxRetry  = 1
-	MaxDirectTry                 = 3
+	PublicIPEchoTimeout        = time.Second * 5
+	NatDetectTimeout           = time.Second * 5
+	UDPReadTimeout             = time.Second * 5
+	ClientAPITimeout           = time.Second * 10
+	UnderlayConnectTimeout     = time.Second * 10
+	MaxDirectTry               = 3
 )
 
 // NATNone has public ip
@@ -170,8 +211,9 @@ const (
 const (
 	LinkModeUDPPunch = "udppunch"
 	LinkModeTCPPunch = "tcppunch"
-	LinkModeIPv4     = "ipv4" // for web
+	LinkModeIPv4     = "ipv4"     // for web
-	LinkModeIPv6     = "ipv6" // for web
+	LinkModeIntranet = "intranet" // for web
+	LinkModeIPv6     = "ipv6"     // for web
 	LinkModeTCP6     = "tcp6"
 	LinkModeTCP4     = "tcp4"
 	LinkModeUDP6     = "udp6"
@@ -179,8 +221,19 @@ const (
 )
 
 const (
-	MsgQueryPeerInfoReq = iota
+	MsgQueryPeerInfoReq = 0
-	MsgQueryPeerInfoRsp
+	MsgQueryPeerInfoRsp = 1
+)
+
+const (
+	MsgSDWANInfoReq = 0
+	MsgSDWANInfoRsp = 1
+)
+
+// MsgNATDetect
+const (
+	MsgNAT      = 0
+	MsgPublicIP = 1
 )
 
 func newMessage(mainType uint16, subType uint16, packet interface{}) ([]byte, error) {
@@ -203,7 +256,22 @@ func newMessage(mainType uint16, subType uint16, packet interface{}) ([]byte, er
 	return writeBytes, nil
 }
 
-func nodeNameToID(name string) uint64 {
+func newMessageWithBuff(mainType uint16, subType uint16, data []byte) ([]byte, error) {
+	head := openP2PHeader{
+		uint32(len(data)),
+		mainType,
+		subType,
+	}
+	headBuf := new(bytes.Buffer)
+	err := binary.Write(headBuf, binary.LittleEndian, head)
+	if err != nil {
+		return nil, err
+	}
+	writeBytes := append(headBuf.Bytes(), data...)
+	return writeBytes, nil
+}
+
+func NodeNameToID(name string) uint64 {
 	return crc64.Checksum([]byte(name), crc64.MakeTable(crc64.ISO))
 }
 
@@ -221,7 +289,11 @@ type PushConnectReq struct {
 	ID               uint64 `json:"id,omitempty"`
 	AppKey           uint64 `json:"appKey,omitempty"` // for underlay tcp
 	LinkMode         string `json:"linkMode,omitempty"`
-	IsUnderlayServer int    `json:"isServer,omitempty"` // Requset spec peer is server
+	IsUnderlayServer int    `json:"isServer,omitempty"`         // Requset spec peer is server
+	UnderlayProtocol string `json:"underlayProtocol,omitempty"` // quic or kcp, default quic
+}
+type PushDstNodeOnline struct {
+	Node string `json:"node,omitempty"`
 }
 type PushConnectRsp struct {
 	Error           int    `json:"error,omitempty"`
@@ -235,6 +307,7 @@ type PushConnectRsp struct {
 	ConeNatPort     int    `json:"coneNatPort,omitempty"` //it's not only cone, but also upnp or nat-pmp hole
 	FromIP          string `json:"fromIP,omitempty"`
 	ID              uint64 `json:"id,omitempty"`
+	PunchTs         uint64 `json:"punchts,omitempty"` // server timestamp
 	Version         string `json:"version,omitempty"`
 }
 type PushRsp struct {
@@ -243,12 +316,15 @@ type PushRsp struct {
 }
 
 type LoginRsp struct {
-	Error  int    `json:"error,omitempty"`
+	Error         int    `json:"error,omitempty"`
-	Detail string `json:"detail,omitempty"`
+	Detail        string `json:"detail,omitempty"`
-	User   string `json:"user,omitempty"`
+	User          string `json:"user,omitempty"`
-	Node   string `json:"node,omitempty"`
+	Node          string `json:"node,omitempty"`
-	Token  uint64 `json:"token,omitempty"`
+	Token         uint64 `json:"token,omitempty"`
-	Ts     int64  `json:"ts,omitempty"`
+	Ts            int64  `json:"ts,omitempty"`
+	LoginMaxDelay int    `json:"loginMaxDelay,omitempty"` // seconds
+	Forcev6       int    `json:"forcev6,omitempty"`
+	PublicIPPort  int    `json:"publicIPPort,omitempty"`
 }
 
 type NatDetectReq struct {
@@ -283,7 +359,8 @@ type TunnelMsg struct {
 }
 
 type RelayNodeReq struct {
-	PeerNode string `json:"peerNode,omitempty"`
+	PeerNode     string `json:"peerNode,omitempty"`
+	ExcludeNodes string `json:"excludeNodes,omitempty"` //TODO: add exclude ip
 }
 
 type RelayNodeRsp struct {
@@ -293,11 +370,15 @@ type RelayNodeRsp struct {
 }
 
 type AddRelayTunnelReq struct {
-	From       string `json:"from,omitempty"`
+	From             string `json:"from,omitempty"`
-	RelayName  string `json:"relayName,omitempty"`
+	RelayName        string `json:"relayName,omitempty"`
-	RelayToken uint64 `json:"relayToken,omitempty"`
+	RelayTunnelID    uint64 `json:"relayTunnelID,omitempty"`
-	AppID      uint64 `json:"appID,omitempty"`  // deprecated
+	RelayToken       uint64 `json:"relayToken,omitempty"`
-	AppKey     uint64 `json:"appKey,omitempty"` // deprecated
+	RelayMode        string `json:"relayMode,omitempty"`
+	AppID            uint64 `json:"appID,omitempty"`            // deprecated
+	AppKey           uint64 `json:"appKey,omitempty"`           // deprecated
+	UnderlayProtocol string `json:"underlayProtocol,omitempty"` // quic or kcp, default quic
+	PunchPriority    int    `json:"punchPriority,omitempty"`
 }
 
 type APPKeySync struct {
@@ -306,8 +387,10 @@ type APPKeySync struct {
 }
 
 type RelayHeartbeat struct {
-	RelayTunnelID uint64 `json:"relayTunnelID,omitempty"`
+	From           string `json:"from,omitempty"`
-	AppID         uint64 `json:"appID,omitempty"`
+	RelayTunnelID  uint64 `json:"relayTunnelID,omitempty"`
+	RelayTunnelID2 uint64 `json:"relayTunnelID2,omitempty"`
+	AppID          uint64 `json:"appID,omitempty"`
 }
 
 type ReportBasic struct {
@@ -316,6 +399,7 @@ type ReportBasic struct {
 	LanIP           string  `json:"lanIP,omitempty"`
 	HasIPv4         int     `json:"hasIPv4,omitempty"`
 	IPv6            string  `json:"IPv6,omitempty"`
+	PublicIPPort    int     `json:"publicIPPort,omitempty"`
 	HasUPNPorNATPMP int     `json:"hasUPNPorNATPMP,omitempty"`
 	Version         string  `json:"version,omitempty"`
 	NetInfo         NetInfo `json:"netInfo,omitempty"`
@@ -341,9 +425,11 @@ type AppInfo struct {
 	AppName        string `json:"appName,omitempty"`
 	Error          string `json:"error,omitempty"`
 	Protocol       string `json:"protocol,omitempty"`
+	PunchPriority  int    `json:"punchPriority,omitempty"`
+	Whitelist      string `json:"whitelist,omitempty"`
 	SrcPort        int    `json:"srcPort,omitempty"`
 	Protocol0      string `json:"protocol0,omitempty"`
-	SrcPort0       int    `json:"srcPort0,omitempty"`
+	SrcPort0       int    `json:"srcPort0,omitempty"` // srcport+protocol is uneque, use as old app id
 	NatType        int    `json:"natType,omitempty"`
 	PeerNode       string `json:"peerNode,omitempty"`
 	DstPort        int    `json:"dstPort,omitempty"`
@@ -353,6 +439,7 @@ type AppInfo struct {
 	PeerIP         string `json:"peerIP,omitempty"`
 	ShareBandwidth int    `json:"shareBandWidth,omitempty"`
 	RelayNode      string `json:"relayNode,omitempty"`
+	SpecRelayNode  string `json:"specRelayNode,omitempty"`
 	RelayMode      string `json:"relayMode,omitempty"`
 	LinkMode       string `json:"linkMode,omitempty"`
 	Version        string `json:"version,omitempty"`
@@ -363,13 +450,16 @@ type AppInfo struct {
 }
 
 type ReportApps struct {
-	Apps []AppInfo
+	Apps     []AppInfo
+	TunError string `json:"tunError,omitempty"`
 }
 
 type ReportLogReq struct {
-	FileName string `json:"fileName,omitempty"`
+	FileName      string `json:"fileName,omitempty"`
-	Offset   int64  `json:"offset,omitempty"`
+	Offset        int64  `json:"offset,omitempty"`
-	Len      int64  `json:"len,omitempty"`
+	Len           int64  `json:"len,omitempty"`
+	IsSetLogLevel int64  `json:"isSetLogLevel,omitempty"`
+	LogLevel      int64  `json:"loglevel,omitempty"`
 }
 type ReportLogRsp struct {
 	FileName string `json:"fileName,omitempty"`
@@ -382,6 +472,7 @@ type UpdateInfo struct {
 	Error       int    `json:"error,omitempty"`
 	ErrorDetail string `json:"errorDetail,omitempty"`
 	Url         string `json:"url,omitempty"`
+	Url2        string `json:"url2,omitempty"`
 }
 
 type NetInfo struct {
@@ -413,8 +504,10 @@ type ProfileInfo struct {
 }
 
 type EditNode struct {
-	NewName   string `json:"newName,omitempty"`
+	NewName      string `json:"newName,omitempty"`
-	Bandwidth int    `json:"bandwidth,omitempty"`
+	Bandwidth    int    `json:"bandwidth,omitempty"`
+	Forcev6      int    `json:"forcev6,omitempty"`
+	PublicIPPort int    `json:"publicIPPort,omitempty"`
 }
 
 type QueryPeerInfoReq struct {
@@ -422,11 +515,218 @@ type QueryPeerInfoReq struct {
 	PeerNode string `json:"peerNode,omitempty"`
 }
 type QueryPeerInfoRsp struct {
+	PeerNode        string `json:"peerNode,omitempty"`
 	Online          int    `json:"online,omitempty"`
 	Version         string `json:"version,omitempty"`
 	NatType         int    `json:"natType,omitempty"`
 	IPv4            string `json:"IPv4,omitempty"`
+	LanIP           string `json:"lanIP,omitempty"`
 	HasIPv4         int    `json:"hasIPv4,omitempty"` // has public ipv4
 	IPv6            string `json:"IPv6,omitempty"`    // if public relay node, ipv6 not set
 	HasUPNPorNATPMP int    `json:"hasUPNPorNATPMP,omitempty"`
 }
+
+type SDWANNode struct {
+	Name     string `json:"name,omitempty"`
+	IP       string `json:"ip,omitempty"`
+	Resource string `json:"resource,omitempty"`
+	Enable   int32  `json:"enable,omitempty"`
+}
+
+type SDWANInfo struct {
+	ID            uint64 `json:"id,omitempty"`
+	Name          string `json:"name,omitempty"`
+	Gateway       string `json:"gateway,omitempty"`
+	Mode          string `json:"mode,omitempty"` // default: fullmesh; central
+	CentralNode   string `json:"centralNode,omitempty"`
+	ForceRelay    int32  `json:"forceRelay,omitempty"`
+	PunchPriority int32  `json:"punchPriority,omitempty"`
+	Enable        int32  `json:"enable,omitempty"`
+	TunnelNum     int32  `json:"tunnelNum,omitempty"`
+	Mtu           int32  `json:"mtu,omitempty"`
+	Nodes         []*SDWANNode
+}
+
+func (s *SDWANInfo) GetResourceByNodeName(nodeName string) string {
+	for _, node := range s.Nodes {
+		if node.Name == nodeName {
+			return node.Resource
+		}
+	}
+	return ""
+}
+
+const (
+	SDWANModeFullmesh = "fullmesh"
+	SDWANModeCentral  = "central"
+)
+
+type ServerSideSaveMemApp struct {
+	From          string `json:"from,omitempty"`
+	Node          string `json:"node,omitempty"`          // for server side findtunnel, maybe relayNode
+	TunnelID      uint64 `json:"tunnelID,omitempty"`      // save in app.tunnel or app.relayTunnel
+	RelayTunnelID uint64 `json:"relayTunnelID,omitempty"` // rtid, if not 0 relay
+	RelayMode     string `json:"relayMode,omitempty"`
+	AppID         uint64 `json:"appID,omitempty"`
+	AppKey        uint64 `json:"appKey,omitempty"`
+	RelayIndex    uint32 `json:"relayIndex,omitempty"`
+	TunnelNum     uint32 `json:"tunnelNum,omitempty"`
+	SrcPort       uint32 `json:"srcPort,omitempty"`
+}
+
+type CheckRemoteService struct {
+	Host string `json:"host,omitempty"`
+	Port uint32 `json:"port,omitempty"`
+}
+
+type SpecTunnel struct {
+	TunnelIndex uint32 `json:"tunnelIndex,omitempty"`
+}
+
+type Nat4Detect struct {
+	CustomData []*Nat4DetectItem
+	Num        int32 `json:"num,omitempty"`
+	*Nat4DetectItem
+}
+
+type Nat4DetectItem struct {
+	Protocol   string `json:"protocol,omitempty"`
+	Server     string `json:"server,omitempty"`
+	ServerPort int32  `json:"serverPort,omitempty"`
+	LocalPort  int32  `json:"localPort,omitempty"`
+}
+
+const rootCA = `-----BEGIN CERTIFICATE-----
+MIIDhTCCAm0CFHm0cd8dnGCbUW/OcS56jf0gvRk7MA0GCSqGSIb3DQEBCwUAMH4x
+CzAJBgNVBAYTAkNOMQswCQYDVQQIDAJHRDETMBEGA1UECgwKb3BlbnAycC5jbjET
+MBEGA1UECwwKb3BlbnAycC5jbjETMBEGA1UEAwwKb3BlbnAycC5jbjEjMCEGCSqG
+SIb3DQEJARYUb3BlbnAycC5jbkBnbWFpbC5jb20wIBcNMjMwODAxMDkwMjMwWhgP
+MjEyMzA3MDgwOTAyMzBaMH4xCzAJBgNVBAYTAkNOMQswCQYDVQQIDAJHRDETMBEG
+A1UECgwKb3BlbnAycC5jbjETMBEGA1UECwwKb3BlbnAycC5jbjETMBEGA1UEAwwK
+b3BlbnAycC5jbjEjMCEGCSqGSIb3DQEJARYUb3BlbnAycC5jbkBnbWFpbC5jb20w
+ggEiMA0GCSqGSIb3DQEBAQUAA4IBDwAwggEKAoIBAQDWg8wPy5hBLUaY4WOXayKu
++magEz1LAY0krzXYSZaSCvGMwA0cervwAqgKfiiZEhho5UNA5iVOJ6bO1RL9H7Vp
+4HuW9BttDU/NQHguD8pyqx06Kaosz5LRw8USz1BCWWFdmi8Mv4I0omtd7m6lbWnY
+nrjQKLYPahPW481jUfJPqR6wUTnBuBMr2ZAGqmFR4Lhqs9B1P9GeBfDWNwVApJUC
+VEhbElukRJxdUvWeJ5+HMENKQcHCTTgmQbmDLMobHXs3Xf7fT9qC76wOe9LFHI6L
+dAww9gryQhxWauQl1NO8aGJTFu+3wgnKBdTMJmF/1iuZYXJOCR1solwqU1hCgBsj
+AgMBAAEwDQYJKoZIhvcNAQELBQADggEBADp153YNVN8p6/3PLnXxHBDeDViAfeQd
+VJmy8eH1LTq/xtUY71HGSpL7iIBNoQdDTHfsg3c6ZANBCxbO/7AhFAzPt1aK8eHy
+XuEiW0Z6R8np1Khh3alCOfD15tKcjok//Wxisbz+YItlbDus/eWRbLGB3HGrzn4l
+GB18jw+G7o4U3rGX8agHqVGQEd06gk1ZaprASpTGwSsv4A5ehosjT1d7re8Z5eD4
+RVtXS+DplMClQ5QSlv3StwcWOsjyiAimNfLEU5xoEfq17yOJUTU1OTL4YOt16QUc
+C1tnzFr3k/ioqFR7cnyzNrbjlfPOmO9l2WReEbMP3bvaSHm6EcpJKS8=
+-----END CERTIFICATE-----`
+
+const rootEdgeCA = `-----BEGIN CERTIFICATE-----
+MIID/zCCAuegAwIBAgIUI53UqyuJSa74NFIKherg5WTjtl4wDQYJKoZIhvcNAQEL
+BQAwgYYxCzAJBgNVBAYTAkNOMQswCQYDVQQIDAJHRDETMBEGA1UECgwKb3BlbnAy
+cC5jbjETMBEGA1UECwwKb3BlbnAycC5jbjEbMBkGA1UEAwwSb3BlbnAycC5jbiBS
+b290IENBMSMwIQYJKoZIhvcNAQkBFhRvcGVucDJwLmNuQGdtYWlsLmNvbTAeFw0y
+NTA5MDMwNTExMTBaFw0zNTA5MDEwNTExMTBaMIGGMQswCQYDVQQGEwJDTjELMAkG
+A1UECAwCR0QxEzARBgNVBAoMCm9wZW5wMnAuY24xEzARBgNVBAsMCm9wZW5wMnAu
+Y24xGzAZBgNVBAMMEm9wZW5wMnAuY24gUm9vdCBDQTEjMCEGCSqGSIb3DQEJARYU
+b3BlbnAycC5jbkBnbWFpbC5jb20wggEiMA0GCSqGSIb3DQEBAQUAA4IBDwAwggEK
+AoIBAQC/aHC0opWx1MFkXYI+Mm0CkMi7nB5XaD3K/DGGtA/kadhayFSWb6Y2+UWW
+s6OYBy7NmQRJgTedS4siQA6JEG4H3FBbz8URLt4TH/EP9+6QB0Z+P0arvUXNkl4k
+7cALmblaiqjq2M199+FWKhDWH2vMr1htY9Y3ldivLRMeH76diKgf8NvsX+wGR8bZ
+4MlJMFln0UeUYKIbekK7DmA5/9f2A/2Nrmi84PKGHU+0ZjB7gik/slW5zH0k7e+S
+wNtTuf8+6+t/LcJK9dWsS6f5+DOWmLcIWs6s/VMP9ODEzlY/hKMFk53+H+AjAZY/
+J/qhOxLXMNlNjdjwSEFPBY/vwVEnAgMBAAGjYzBhMB0GA1UdDgQWBBTXSSeIvz/R
+6A1pz0H4xBlV1Vu9kTAfBgNVHSMEGDAWgBTXSSeIvz/R6A1pz0H4xBlV1Vu9kTAP
+BgNVHRMBAf8EBTADAQH/MA4GA1UdDwEB/wQEAwIBhjANBgkqhkiG9w0BAQsFAAOC
+AQEABqvvKwM+k2NfIFf9tzo1EsD4rQunyn6K5Zhf/kspb9++2Onw/lDlOErxSLLz
+C5aXn+B48honQeYEL/cYhH4duVQb0Zk71iF/PKDxYvF79Xbx9k7Kzg6RryaH8ZfQ
+pyEao+Uc6O895F+SLBog5aHIbz8gFNCRVaSAv3xpUIyQ/haxyHHapaLqt/ueNFVP
+qEG+9R41q55rEYb2ltINhumS3gb4qOcKI5pHuAw42pF8SShqaBIfFXSZ4u9ib7/k
+CvHN0kDYavV6NRiCSRF6wMxmaF70WpfqQhGdw0WyIzJfMOtSdvctjfNCoaWy2V2s
+nLaJXgiPehxIVGNC9dk/ZZzI2g==
+-----END CERTIFICATE-----`
+
+const ISRGRootX1 = `-----BEGIN CERTIFICATE-----
+MIIEJjCCAw6gAwIBAgISAztStWq026ej0RCsk3ErbUdPMA0GCSqGSIb3DQEBCwUA
+MDIxCzAJBgNVBAYTAlVTMRYwFAYDVQQKEw1MZXQncyBFbmNyeXB0MQswCQYDVQQD
+EwJSMzAeFw0yMzA4MDQwODUyMjlaFw0yMzExMDIwODUyMjhaMBcxFTATBgNVBAMM
+DCoub3BlbnAycC5jbjBZMBMGByqGSM49AgEGCCqGSM49AwEHA0IABPRdkgLV2FA+
+3g/GjcA9UcfDfIFYgofSTNbOCQFIiQVMXrTgAToF1/tWaS2LOuysZcCX6OE7SCeG
+lQ+0g+L2qvujggIaMIICFjAOBgNVHQ8BAf8EBAMCB4AwHQYDVR0lBBYwFAYIKwYB
+BQUHAwEGCCsGAQUFBwMCMAwGA1UdEwEB/wQCMAAwHQYDVR0OBBYEFIdL5LNQC+X4
+8r6u+3NlM238Vmk5MB8GA1UdIwQYMBaAFBQusxe3WFbLrlAJQOYfr52LFMLGMFUG
+CCsGAQUFBwEBBEkwRzAhBggrBgEFBQcwAYYVaHR0cDovL3IzLm8ubGVuY3Iub3Jn
+MCIGCCsGAQUFBzAChhZodHRwOi8vcjMuaS5sZW5jci5vcmcvMCMGA1UdEQQcMBqC
+DCoub3BlbnAycC5jboIKb3BlbnAycC5jbjATBgNVHSAEDDAKMAgGBmeBDAECATCC
+AQQGCisGAQQB1nkCBAIEgfUEgfIA8AB2AHoyjFTYty22IOo44FIe6YQWcDIThU07
+0ivBOlejUutSAAABib/2fCgAAAQDAEcwRQIhAJzf9XNe0cu9CNYLLqtDCZZMqI6u
+qsHrnnXcFQW23ioZAiAgwKp5DwZw9RmF19KOjD6lYJfTxc+anJUuWAlMwu1HYQB2
+AK33vvp8/xDIi509nB4+GGq0Zyldz7EMJMqFhjTr3IKKAAABib/2fEEAAAQDAEcw
+RQIgKeI7DopyzFXPdRQZKZrHVqfXQ8OipvlKXd5xRnKFjH4CIQDMM+TU+LOux8xK
+1NlTiSs9DhQI/eU3ZXKxSQAqF50RnTANBgkqhkiG9w0BAQsFAAOCAQEATqZ+H2NT
+cv4FzArD/Krlnur1OTitvpubRWM+ClB9Cr6pvPVB7Dp0/ALxu35ZmCtrzdJWTfmp
+lHxU4nPXRPVjuPRNXooSyH//KTfHyf32919PQOi/qc/QEAuIzkGLJg0dIPKLxaNK
+CiTWU+2iAYSHBgCWulfLX/RYNbBZQ9w0xIm3XhuMjCF/omG8ofuz1DmiRVR+17JA
+nuDXQkxm7KhmbxSA4PsLwzvIWA8Wk44ZK7uncgRY3WIUXcVRELSFA5LuH67TOwag
+al6iG56KW1N2Yy9YmeG27SYvHZYkjmuJ8NEy7Ku+Mi6gwO4hs0CYr2wtUacPfjKF
+aYTGWSt6Pt8kmw==
+-----END CERTIFICATE-----
+-----BEGIN CERTIFICATE-----
+MIIFFjCCAv6gAwIBAgIRAJErCErPDBinU/bWLiWnX1owDQYJKoZIhvcNAQELBQAw
+TzELMAkGA1UEBhMCVVMxKTAnBgNVBAoTIEludGVybmV0IFNlY3VyaXR5IFJlc2Vh
+cmNoIEdyb3VwMRUwEwYDVQQDEwxJU1JHIFJvb3QgWDEwHhcNMjAwOTA0MDAwMDAw
+WhcNMjUwOTE1MTYwMDAwWjAyMQswCQYDVQQGEwJVUzEWMBQGA1UEChMNTGV0J3Mg
+RW5jcnlwdDELMAkGA1UEAxMCUjMwggEiMA0GCSqGSIb3DQEBAQUAA4IBDwAwggEK
+AoIBAQC7AhUozPaglNMPEuyNVZLD+ILxmaZ6QoinXSaqtSu5xUyxr45r+XXIo9cP
+R5QUVTVXjJ6oojkZ9YI8QqlObvU7wy7bjcCwXPNZOOftz2nwWgsbvsCUJCWH+jdx
+sxPnHKzhm+/b5DtFUkWWqcFTzjTIUu61ru2P3mBw4qVUq7ZtDpelQDRrK9O8Zutm
+NHz6a4uPVymZ+DAXXbpyb/uBxa3Shlg9F8fnCbvxK/eG3MHacV3URuPMrSXBiLxg
+Z3Vms/EY96Jc5lP/Ooi2R6X/ExjqmAl3P51T+c8B5fWmcBcUr2Ok/5mzk53cU6cG
+/kiFHaFpriV1uxPMUgP17VGhi9sVAgMBAAGjggEIMIIBBDAOBgNVHQ8BAf8EBAMC
+AYYwHQYDVR0lBBYwFAYIKwYBBQUHAwIGCCsGAQUFBwMBMBIGA1UdEwEB/wQIMAYB
+Af8CAQAwHQYDVR0OBBYEFBQusxe3WFbLrlAJQOYfr52LFMLGMB8GA1UdIwQYMBaA
+FHm0WeZ7tuXkAXOACIjIGlj26ZtuMDIGCCsGAQUFBwEBBCYwJDAiBggrBgEFBQcw
+AoYWaHR0cDovL3gxLmkubGVuY3Iub3JnLzAnBgNVHR8EIDAeMBygGqAYhhZodHRw
+Oi8veDEuYy5sZW5jci5vcmcvMCIGA1UdIAQbMBkwCAYGZ4EMAQIBMA0GCysGAQQB
+gt8TAQEBMA0GCSqGSIb3DQEBCwUAA4ICAQCFyk5HPqP3hUSFvNVneLKYY611TR6W
+PTNlclQtgaDqw+34IL9fzLdwALduO/ZelN7kIJ+m74uyA+eitRY8kc607TkC53wl
+ikfmZW4/RvTZ8M6UK+5UzhK8jCdLuMGYL6KvzXGRSgi3yLgjewQtCPkIVz6D2QQz
+CkcheAmCJ8MqyJu5zlzyZMjAvnnAT45tRAxekrsu94sQ4egdRCnbWSDtY7kh+BIm
+lJNXoB1lBMEKIq4QDUOXoRgffuDghje1WrG9ML+Hbisq/yFOGwXD9RiX8F6sw6W4
+avAuvDszue5L3sz85K+EC4Y/wFVDNvZo4TYXao6Z0f+lQKc0t8DQYzk1OXVu8rp2
+yJMC6alLbBfODALZvYH7n7do1AZls4I9d1P4jnkDrQoxB3UqQ9hVl3LEKQ73xF1O
+yK5GhDDX8oVfGKF5u+decIsH4YaTw7mP3GFxJSqv3+0lUFJoi5Lc5da149p90Ids
+hCExroL1+7mryIkXPeFM5TgO9r0rvZaBFOvV2z0gp35Z0+L4WPlbuEjN/lxPFin+
+HlUjr8gRsI3qfJOQFy/9rKIJR0Y/8Omwt/8oTWgy1mdeHmmjk7j1nYsvC9JSQ6Zv
+MldlTTKB3zhThV1+XWYp6rjd5JW1zbVWEkLNxE7GJThEUG3szgBVGP7pSWTUTsqX
+nLRbwHOoq7hHwg==
+-----END CERTIFICATE-----
+-----BEGIN CERTIFICATE-----
+MIIFazCCA1OgAwIBAgIRAIIQz7DSQONZRGPgu2OCiwAwDQYJKoZIhvcNAQELBQAw
+TzELMAkGA1UEBhMCVVMxKTAnBgNVBAoTIEludGVybmV0IFNlY3VyaXR5IFJlc2Vh
+cmNoIEdyb3VwMRUwEwYDVQQDEwxJU1JHIFJvb3QgWDEwHhcNMTUwNjA0MTEwNDM4
+WhcNMzUwNjA0MTEwNDM4WjBPMQswCQYDVQQGEwJVUzEpMCcGA1UEChMgSW50ZXJu
+ZXQgU2VjdXJpdHkgUmVzZWFyY2ggR3JvdXAxFTATBgNVBAMTDElTUkcgUm9vdCBY
+MTCCAiIwDQYJKoZIhvcNAQEBBQADggIPADCCAgoCggIBAK3oJHP0FDfzm54rVygc
+h77ct984kIxuPOZXoHj3dcKi/vVqbvYATyjb3miGbESTtrFj/RQSa78f0uoxmyF+
+0TM8ukj13Xnfs7j/EvEhmkvBioZxaUpmZmyPfjxwv60pIgbz5MDmgK7iS4+3mX6U
+A5/TR5d8mUgjU+g4rk8Kb4Mu0UlXjIB0ttov0DiNewNwIRt18jA8+o+u3dpjq+sW
+T8KOEUt+zwvo/7V3LvSye0rgTBIlDHCNAymg4VMk7BPZ7hm/ELNKjD+Jo2FR3qyH
+B5T0Y3HsLuJvW5iB4YlcNHlsdu87kGJ55tukmi8mxdAQ4Q7e2RCOFvu396j3x+UC
+B5iPNgiV5+I3lg02dZ77DnKxHZu8A/lJBdiB3QW0KtZB6awBdpUKD9jf1b0SHzUv
+KBds0pjBqAlkd25HN7rOrFleaJ1/ctaJxQZBKT5ZPt0m9STJEadao0xAH0ahmbWn
+OlFuhjuefXKnEgV4We0+UXgVCwOPjdAvBbI+e0ocS3MFEvzG6uBQE3xDk3SzynTn
+jh8BCNAw1FtxNrQHusEwMFxIt4I7mKZ9YIqioymCzLq9gwQbooMDQaHWBfEbwrbw
+qHyGO0aoSCqI3Haadr8faqU9GY/rOPNk3sgrDQoo//fb4hVC1CLQJ13hef4Y53CI
+rU7m2Ys6xt0nUW7/vGT1M0NPAgMBAAGjQjBAMA4GA1UdDwEB/wQEAwIBBjAPBgNV
+HRMBAf8EBTADAQH/MB0GA1UdDgQWBBR5tFnme7bl5AFzgAiIyBpY9umbbjANBgkq
+hkiG9w0BAQsFAAOCAgEAVR9YqbyyqFDQDLHYGmkgJykIrGF1XIpu+ILlaS/V9lZL
+ubhzEFnTIZd+50xx+7LSYK05qAvqFyFWhfFQDlnrzuBZ6brJFe+GnY+EgPbk6ZGQ
+3BebYhtF8GaV0nxvwuo77x/Py9auJ/GpsMiu/X1+mvoiBOv/2X/qkSsisRcOj/KK
+NFtY2PwByVS5uCbMiogziUwthDyC3+6WVwW6LLv3xLfHTjuCvjHIInNzktHCgKQ5
+ORAzI4JMPJ+GslWYHb4phowim57iaztXOoJwTdwJx4nLCgdNbOhdjsnvzqvHu7Ur
+TkXWStAmzOVyyghqpZXjFaH3pO3JLF+l+/+sKAIuvtd7u+Nxe5AW0wdeRlN8NwdC
+jNPElpzVmbUq4JUagEiuTDkHzsxHpFKVK7q4+63SM1N95R1NbdWhscdCb+ZAJzVc
+oyi3B43njTOQ5yOf+1CceWxG1bQVs5ZufpsMljq4Ui0/1lvh+wjChP4kqKOJ2qxq
+4RgqsahDYVvTH9w7jXbyLeiNdd8XM2w9U/t7y0Ff/9yi0GE44Za4rF2LN9d11TPA
+mRGunUHBcnWEvgJBQl9nJEiU0Zsnvgc/ubhPgXRR4Xq37Z0j4r7g1SgEEzwxA57d
+emyPxgcYxn/eR44/KJ4EBs+lVDR3veyJm+kXQ99b21/+jh5Xos1AnX5iItreGCc=
+-----END CERTIFICATE-----
+`

core/sdwan.go

@@ -0,0 +1,383 @@
+package openp2p
+
+import (
+	"encoding/binary"
+	"encoding/json"
+	"fmt"
+	"net"
+	"reflect"
+	"runtime"
+	"strings"
+	"sync"
+	"time"
+)
+
+type PacketHeader struct {
+	version int
+	// src     uint32
+	// prot    uint8
+	protocol byte
+	dst      uint32
+	port     uint16
+}
+
+func parseHeader(b []byte, h *PacketHeader) error {
+	if len(b) < 20 {
+		return fmt.Errorf("small packet")
+	}
+	h.version = int(b[0] >> 4)
+	h.protocol = byte(b[9])
+	if h.version == 4 {
+		h.dst = binary.BigEndian.Uint32(b[16:20])
+	} else if h.version != 6 {
+		return fmt.Errorf("unknown version in ip header:%d", h.version)
+	}
+	if h.protocol == 6 || h.protocol == 17 { // TCP or UDP
+		h.port = binary.BigEndian.Uint16(b[22:24])
+	}
+	return nil
+}
+
+type sdwanNode struct {
+	name string
+	id   uint64
+}
+
+type p2pSDWAN struct {
+	tun           *optun
+	tunErr        string
+	sysRoute      sync.Map // ip:sdwanNode
+	subnet        *net.IPNet
+	gateway       net.IP
+	virtualIP     *net.IPNet
+	internalRoute *IPTree
+}
+
+func (s *p2pSDWAN) reset() {
+	gLog.i("reset sdwan when network disconnected")
+	// clear sysroute
+	delRoutesByGateway(s.gateway.String())
+	s.sysRoute.Range(func(key, value interface{}) bool {
+		s.sysRoute.Delete(key)
+		return true
+	})
+	// clear internel route
+	s.internalRoute = NewIPTree("")
+	// clear p2papp
+	for _, node := range gConf.getSDWAN().Nodes {
+		gConf.delete(AppConfig{SrcPort: 0, PeerNode: node.Name})
+		GNetwork.DeleteApp(AppConfig{SrcPort: 0, PeerNode: node.Name})
+	}
+
+	gConf.resetSDWAN()
+}
+
+func (s *p2pSDWAN) init() error {
+	gConf.Network.previousIP = gConf.Network.publicIP
+	if gConf.getSDWAN().Gateway == "" {
+		gLog.d("sdwan init: not in sdwan clear all ")
+	}
+	if s.internalRoute == nil {
+		s.internalRoute = NewIPTree("")
+	}
+
+	if gw, sn, err := net.ParseCIDR(gConf.getSDWAN().Gateway); err == nil { // preserve old gateway
+		s.gateway = gw
+		s.subnet = sn
+	}
+
+	for _, node := range gConf.getDelNodes() {
+		gLog.d("sdwan init: deal deleted node: %s", node.Name)
+		gLog.d("sdwan init: delRoute: %s, %s ", node.IP, s.gateway.String())
+		// delRoute(node.IP, s.gateway.String()) // TODO: seems no need delelte each node
+		s.internalRoute.Del(node.IP, node.IP)
+		ipNum, _ := inetAtoN(node.IP)
+		s.sysRoute.Delete(ipNum)
+		// if node.Name == gConf.Network.Node {
+		// 	// this is local node, need rm all client-side apps
+		// 	GNetwork.apps.Range(func(id, i interface{}) bool {
+		// 		app := i.(*p2pApp)
+		// 		if app.config.is
+		// 		return true
+		// 	})
+		// 	continue
+		// }
+		gConf.delete(AppConfig{SrcPort: 0, PeerNode: node.Name})
+		GNetwork.DeleteApp(AppConfig{SrcPort: 0, PeerNode: node.Name})
+		arr := strings.Split(node.Resource, ",")
+		for _, r := range arr {
+			_, ipnet, err := net.ParseCIDR(r)
+			if err != nil {
+				// fmt.Println("Error parsing CIDR:", err)
+				continue
+			}
+			if ipnet.Contains(net.ParseIP(gConf.Network.localIP)) { // local ip and resource in the same lan
+				continue
+			}
+			minIP := ipnet.IP
+			maxIP := make(net.IP, len(minIP))
+			copy(maxIP, minIP)
+			for i := range minIP {
+				maxIP[i] = minIP[i] | ^ipnet.Mask[i]
+			}
+			s.internalRoute.Del(minIP.String(), maxIP.String())
+			delRoute(ipnet.String(), s.gateway.String())
+			gLog.d("sdwan init: resource delRoute: %s, %s ", ipnet.String(), s.gateway.String())
+		}
+	}
+	for _, node := range gConf.getAddNodes() {
+		gLog.d("sdwan init: deal add node: %s", node.Name)
+		ipNet := &net.IPNet{
+			IP:   net.ParseIP(node.IP),
+			Mask: s.subnet.Mask,
+		}
+		if node.Name == gConf.Network.Node {
+			s.virtualIP = ipNet
+			gLog.i("sdwan init: start tun %s", ipNet.String())
+			err := s.StartTun()
+			if err != nil {
+				gLog.e("sdwan init: start tun error:%s", err)
+				return err
+			}
+			gLog.i("sdwan init: start tun ok")
+			allowTunForward()
+			gLog.d("sdwan init: addRoute %s %s %s", s.subnet.String(), s.gateway.String(), s.tun.tunName)
+			addRoute(s.subnet.String(), s.gateway.String(), s.tun.tunName)
+			// addRoute("255.255.255.255/32", s.gateway.String(), s.tun.tunName) // for broadcast
+			// addRoute("224.0.0.0/4", s.gateway.String(), s.tun.tunName)        // for multicast
+			initSNATRule(s.subnet.String()) // for network resource
+			continue
+		}
+		ip, err := inetAtoN(ipNet.String())
+		if err != nil {
+			return err
+		}
+		s.sysRoute.Store(ip, &sdwanNode{name: node.Name, id: NodeNameToID(node.Name)})
+		s.internalRoute.AddIntIP(ip, ip, &sdwanNode{name: node.Name, id: NodeNameToID(node.Name)})
+	}
+	for _, node := range gConf.getAddNodes() {
+		if node.Name == gConf.Network.Node { // not deal resource itself
+			continue
+		}
+		if len(node.Resource) > 0 {
+			gLog.i("sdwan init: deal add node: %s resource: %s", node.Name, node.Resource)
+			arr := strings.Split(node.Resource, ",")
+			for _, r := range arr {
+				// add internal route
+				_, ipnet, err := net.ParseCIDR(r)
+				if err != nil {
+					fmt.Println("sdwan init: Error parsing CIDR:", err)
+					continue
+				}
+				if ipnet.Contains(net.ParseIP(gConf.Network.localIP)) { // local ip and resource in the same lan
+					gLog.d("sdwan init: local ip %s in this resource %s, ignore", gConf.Network.localIP, ipnet.IP.String())
+					continue
+				}
+				// local net could access this single ip
+				if ipnet.Mask[0] == 255 && ipnet.Mask[1] == 255 && ipnet.Mask[2] == 255 && ipnet.Mask[3] == 255 {
+					gLog.d("sdwan init: ping %s start", ipnet.IP.String())
+					if _, err := Ping(ipnet.IP.String()); err == nil {
+						gLog.d("sdwan init: ping %s ok, ignore this resource", ipnet.IP.String())
+						continue
+					}
+					gLog.d("sdwan init: ping %s failed", ipnet.IP.String())
+				}
+				minIP := ipnet.IP
+				maxIP := make(net.IP, len(minIP))
+				copy(maxIP, minIP)
+				for i := range minIP {
+					maxIP[i] = minIP[i] | ^ipnet.Mask[i]
+				}
+				s.internalRoute.Add(minIP.String(), maxIP.String(), &sdwanNode{name: node.Name, id: NodeNameToID(node.Name)})
+				// add sys route
+				gLog.d("sdwan init: addRoute %s %s %s", ipnet.String(), s.gateway.String(), s.tun.tunName)
+				addRoute(ipnet.String(), s.gateway.String(), s.tun.tunName)
+			}
+		}
+	}
+	gConf.retryAllMemApp()
+	gLog.i("sdwan init ok")
+	return nil
+}
+
+func (s *p2pSDWAN) run() {
+	s.sysRoute.Range(func(key, value interface{}) bool {
+		node := value.(*sdwanNode)
+		GNetwork.ConnectNode(node.name)
+		return true
+	})
+}
+
+func (s *p2pSDWAN) readNodeLoop() {
+	gLog.d("sdwan readNodeLoop start")
+	defer gLog.d("sdwan readNodeLoop end")
+	writeBuff := make([][]byte, 1)
+	for {
+		nd := GNetwork.ReadNode(time.Second * 10) // TODO: read multi packet
+		if nd == nil {
+			gLog.dev("waiting for node data")
+			continue
+		}
+		head := PacketHeader{}
+		parseHeader(nd, &head)
+		gLog.dev("write tun dst ip=%s,len=%d", net.IP{byte(head.dst >> 24), byte(head.dst >> 16), byte(head.dst >> 8), byte(head.dst)}.String(), len(nd))
+		if PIHeaderSize == 0 {
+			writeBuff[0] = nd
+		} else {
+			writeBuff[0] = make([]byte, PIHeaderSize+len(nd))
+			copy(writeBuff[0][PIHeaderSize:], nd)
+		}
+
+		len, err := s.tun.Write(writeBuff, PIHeaderSize)
+		if err != nil {
+			gLog.d("write tun dst ip=%s,len=%d,error:%s", net.IP{byte(head.dst >> 24), byte(head.dst >> 16), byte(head.dst >> 8), byte(head.dst)}.String(), len, err)
+		}
+	}
+}
+
+func isBroadcastOrMulticast(ipUint32 uint32, subnet *net.IPNet) bool {
+	// return ipUint32 == 0xffffffff || (byte(ipUint32) == 0xff) || (ipUint32>>28 == 0xe)
+	return ipUint32 == 0xffffffff || (ipUint32>>28 == 0xe) // 225.255.255.255/32, 224.0.0.0/4
+}
+
+func (s *p2pSDWAN) routeTunPacket(p []byte, head *PacketHeader) {
+	var node *sdwanNode
+	// v, ok := s.routes.Load(ih.dst)
+	v, ok := s.internalRoute.Load(head.dst)
+	if !ok || v == nil {
+		if isBroadcastOrMulticast(head.dst, s.subnet) {
+			gLog.dev("multicast ip=%s", net.IP{byte(head.dst >> 24), byte(head.dst >> 16), byte(head.dst >> 8), byte(head.dst)}.String())
+			GNetwork.WriteBroadcast(p)
+			return
+		}
+		gLog.dev("internalRoute not found ip:%s", net.IP{byte(head.dst >> 24), byte(head.dst >> 16), byte(head.dst >> 8), byte(head.dst)}.String())
+		return
+	} else {
+		node = v.(*sdwanNode)
+	}
+
+	err := GNetwork.WriteNode(node.id, p)
+	if err != nil {
+		gLog.dev("write packet to %s fail: %s", node.name, err)
+	}
+}
+
+func (s *p2pSDWAN) readTunLoop() {
+	gLog.d("sdwan readTunLoop start")
+	defer gLog.d("sdwan readTunLoop end")
+	readBuff := make([][]byte, ReadTunBuffNum)
+	for i := 0; i < ReadTunBuffNum; i++ {
+		readBuff[i] = make([]byte, ReadTunBuffSize+PIHeaderSize)
+	}
+	readBuffSize := make([]int, ReadTunBuffNum)
+	ih := PacketHeader{}
+	for {
+		n, err := s.tun.Read(readBuff, readBuffSize, PIHeaderSize)
+		if err != nil {
+			gLog.e("read tun fail: %s", err)
+			return
+		}
+		for i := 0; i < n; i++ {
+			if readBuffSize[i] > ReadTunBuffSize {
+				gLog.e("read tun overflow: len=%d", readBuffSize[i])
+				continue
+			}
+			parseHeader(readBuff[i][PIHeaderSize:readBuffSize[i]+PIHeaderSize], &ih)
+			gLog.dev("read tun dst ip=%s,len=%d", net.IP{byte(ih.dst >> 24), byte(ih.dst >> 16), byte(ih.dst >> 8), byte(ih.dst)}.String(), readBuffSize[0])
+			s.routeTunPacket(readBuff[i][PIHeaderSize:readBuffSize[i]+PIHeaderSize], &ih)
+		}
+	}
+}
+
+func (s *p2pSDWAN) StartTun() error {
+	sdwan := gConf.getSDWAN()
+	if s.tun == nil {
+		tun := &optun{}
+		err := tun.Start(s.virtualIP.String(), &sdwan)
+		if err != nil {
+			gLog.e("open tun fail:%v", err)
+			s.tunErr = err.Error()
+			return err
+		}
+		s.tun = tun
+		s.tunErr = ""
+		go s.readTunLoop()
+		go s.readNodeLoop() // multi-thread read will cause packets out of order, resulting in slower speeds
+	}
+	err := setTunAddr(s.tun.tunName, s.virtualIP.String(), sdwan.Gateway, s.tun.dev)
+	if err != nil {
+		gLog.e("setTunAddr error:%s,%s,%s,%s", err, s.tun.tunName, s.virtualIP.String(), sdwan.Gateway)
+		return err
+	}
+	return nil
+}
+
+func handleSDWAN(subType uint16, msg []byte) error {
+	gLog.d("handle sdwan msg type:%d", subType)
+	var err error
+	switch subType {
+	case MsgSDWANInfoRsp:
+		rsp := SDWANInfo{}
+		if err = json.Unmarshal(msg[openP2PHeaderSize:], &rsp); err != nil {
+			return ErrMsgFormat
+		}
+		gLog.i("sdwan init:%s", prettyJson(rsp))
+		// GNetwork.sdwan.detail = &rsp
+		if gConf.Network.previousIP != gConf.Network.publicIP || gConf.getSDWAN().CentralNode != rsp.CentralNode || gConf.getSDWAN().Gateway != rsp.Gateway {
+			GNetwork.sdwan.reset()
+			preAndroidSDWANConfig = "" // let androind app reset vpnservice
+		}
+		gConf.setSDWAN(rsp)
+		if runtime.GOOS == "android" {
+			if !compareResources(preAndroidSDWANConfig, string(msg[openP2PHeaderSize:])) { // when config change, notify android app
+				select {
+				case AndroidSDWANConfig <- msg[openP2PHeaderSize:]:
+				default:
+				}
+				preAndroidSDWANConfig = string(msg[openP2PHeaderSize:])
+			}
+		}
+		err = GNetwork.sdwan.init()
+		if err != nil {
+			gLog.e("sdwan init fail: %s", err)
+			if GNetwork.sdwan.tun != nil {
+				GNetwork.sdwan.tun.Stop()
+				GNetwork.sdwan.tun = nil
+				return err
+			}
+		}
+		go GNetwork.sdwan.run()
+	default:
+	}
+	return err
+}
+
+// for android vpnservice
+func compareResources(json1, json2 string) bool {
+	var net1, net2 SDWANInfo
+	if err := json.Unmarshal([]byte(json1), &net1); err != nil {
+		fmt.Println("Error parsing json1:", err)
+		fmt.Println("Error parsing json1:", string(json1))
+		return false
+	}
+	if err := json.Unmarshal([]byte(json2), &net2); err != nil {
+		fmt.Println("Error parsing json2:", err)
+		fmt.Println("Error parsing json1:", string(json2))
+		return false
+	}
+
+	// 获取所有资源并比较
+	resources1 := getResources(net1)
+	resources2 := getResources(net2)
+	return reflect.DeepEqual(resources1, resources2)
+}
+
+func getResources(network SDWANInfo) []string {
+	var resources []string
+	for _, node := range network.Nodes {
+		if node.Resource != "" {
+			resources = append(resources, node.Resource)
+		}
+	}
+	return resources
+}

core/speedlimiter.go

@@ -0,0 +1,50 @@
+package openp2p
+
+import (
+	"sync"
+	"time"
+)
+
+// SpeedLimiter ...
+type SpeedLimiter struct {
+	lastUpdate time.Time
+	speed      int // per second
+	precision  int // seconds
+	freeCap    int
+	maxFreeCap int
+	mtx        sync.Mutex
+}
+
+func newSpeedLimiter(speed int, precision int) *SpeedLimiter {
+	return &SpeedLimiter{
+		speed:      speed,
+		precision:  precision,
+		lastUpdate: time.Now(),
+		maxFreeCap: speed * precision,
+		freeCap:    speed * precision,
+	}
+}
+
+// Add ...
+func (sl *SpeedLimiter) Add(increment int, wait bool) bool {
+	if sl.speed <= 0 {
+		return true
+	}
+	sl.mtx.Lock()
+	defer sl.mtx.Unlock()
+	sl.freeCap += int(time.Since(sl.lastUpdate) * time.Duration(sl.speed) / time.Second)
+	if sl.freeCap > sl.maxFreeCap {
+		sl.freeCap = sl.maxFreeCap
+	}
+	if !wait && sl.freeCap < increment {
+		return false
+	}
+	sl.freeCap -= increment
+	sl.lastUpdate = time.Now()
+	if sl.freeCap < 0 {
+		// sleep for the overflow
+		// fmt.Println("sleep ", time.Millisecond*time.Duration(-sl.freeCap*100)/time.Duration(sl.speed))
+		time.Sleep(time.Millisecond * time.Duration(-sl.freeCap*1000) / time.Duration(sl.speed)) // sleep ms
+	}
+	return true
+}

core/speedlimiter_test.go

@@ -0,0 +1,61 @@
+package openp2p
+
+import (
+	"log"
+	"testing"
+	"time"
+)
+
+func TestBandwidth(t *testing.T) {
+	speed := 10 * 1024 * 1024 / 8 // 10mbps
+	speedl := newSpeedLimiter(speed, 1)
+	oneBuffSize := 4096
+	writeNum := 5000
+	expectTime := oneBuffSize * writeNum / speed
+	startTs := time.Now()
+	for i := 0; i < writeNum; i++ {
+		speedl.Add(oneBuffSize, true)
+	}
+	log.Printf("cost %ds, expect %ds", time.Since(startTs)/time.Second, expectTime)
+	if time.Since(startTs) > time.Duration(expectTime+1)*time.Second || time.Since(startTs) < time.Duration(expectTime-1)*time.Second {
+		t.Error("error")
+	}
+}
+
+func TestSymmetric(t *testing.T) {
+	speed := 20000 / 180
+	speedl := newSpeedLimiter(speed, 180)
+	oneBuffSize := 300
+	writeNum := 70
+	expectTime := (oneBuffSize*writeNum - 20000) / speed
+	log.Printf("expect %ds", expectTime)
+	startTs := time.Now()
+	for i := 0; i < writeNum; i++ {
+		speedl.Add(oneBuffSize, true)
+	}
+	log.Printf("cost %ds, expect %ds", time.Since(startTs)/time.Second, expectTime)
+	if time.Since(startTs) > time.Duration(expectTime+1)*time.Second || time.Since(startTs) < time.Duration(expectTime-1)*time.Second {
+		t.Error("error")
+	}
+}
+
+func TestSymmetric2(t *testing.T) {
+	speed := 30000 / 180
+	speedl := newSpeedLimiter(speed, 180)
+	oneBuffSize := 800
+	writeNum := 40
+	expectTime := (oneBuffSize*writeNum - 30000) / speed
+	log.Printf("expect %ds", expectTime)
+	startTs := time.Now()
+	for i := 0; i < writeNum; {
+		if speedl.Add(oneBuffSize, true) {
+			i++
+		} else {
+			time.Sleep(time.Millisecond)
+		}
+	}
+	log.Printf("cost %ds, expect %ds", time.Since(startTs)/time.Second, expectTime)
+	if time.Since(startTs) > time.Duration(expectTime+1)*time.Second || time.Since(startTs) < time.Duration(expectTime-1)*time.Second {
+		t.Error("error")
+	}
+}

core/totp.go

@@ -1,35 +0,0 @@
-// Time-based One-time Password
-package openp2p
-
-import (
-	"crypto/hmac"
-	"crypto/sha256"
-	"encoding/binary"
-)
-
-const TOTPStep = 30 // 30s
-func GenTOTP(token uint64, ts int64) uint64 {
-	step := ts / TOTPStep
-	tbuff := make([]byte, 8)
-	binary.LittleEndian.PutUint64(tbuff, token)
-	mac := hmac.New(sha256.New, tbuff)
-	b := make([]byte, 8)
-	binary.LittleEndian.PutUint64(b, uint64(step))
-	mac.Write(b)
-	num := binary.LittleEndian.Uint64(mac.Sum(nil)[:8])
-	// fmt.Printf("%x\n", mac.Sum(nil))
-	return num
-}
-
-func VerifyTOTP(code uint64, token uint64, ts int64) bool {
-	if code == 0 {
-		return false
-	}
-	if code == token {
-		return true
-	}
-	if code == GenTOTP(token, ts) || code == GenTOTP(token, ts-TOTPStep) || code == GenTOTP(token, ts+TOTPStep) {
-		return true
-	}
-	return false
-}

core/totp_test.go

@@ -1,36 +0,0 @@
-// Time-based One-time Password
-package openp2p
-
-import (
-	"testing"
-	"time"
-)
-
-func TestTOTP(t *testing.T) {
-	for i := 0; i < 20; i++ {
-		ts := time.Now().Unix()
-		code := GenTOTP(13666999958022769123, ts)
-		t.Log(code)
-		if !VerifyTOTP(code, 13666999958022769123, ts) {
-			t.Error("TOTP error")
-		}
-		if !VerifyTOTP(code, 13666999958022769123, ts-10) {
-			t.Error("TOTP error")
-		}
-		if !VerifyTOTP(code, 13666999958022769123, ts+10) {
-			t.Error("TOTP error")
-		}
-		if VerifyTOTP(code, 13666999958022769123, ts+60) {
-			t.Error("TOTP error")
-		}
-		if VerifyTOTP(code, 13666999958022769124, ts+1) {
-			t.Error("TOTP error")
-		}
-		if VerifyTOTP(code, 13666999958022769125, ts+1) {
-			t.Error("TOTP error")
-		}
-		time.Sleep(time.Second)
-		t.Log("round", i, " ", ts, " test ok")
-	}
-
-}

core/udp.go

@@ -18,27 +18,30 @@ func UDPWrite(conn *net.UDPConn, dst net.Addr, mainType uint16, subType uint16,
 	return conn.WriteTo(msg, dst)
 }
 
-func UDPRead(conn *net.UDPConn, timeout int) (ra net.Addr, head *openP2PHeader, result []byte, len int, err error) {
+func UDPRead(conn *net.UDPConn, timeout time.Duration) (ra net.Addr, head *openP2PHeader, buff []byte, length int, err error) {
 	if timeout > 0 {
-		deadline := time.Now().Add(time.Millisecond * time.Duration(timeout))
+		err = conn.SetReadDeadline(time.Now().Add(timeout))
-		err = conn.SetReadDeadline(deadline)
 		if err != nil {
-			gLog.Println(LvERROR, "SetReadDeadline error")
+			gLog.e("SetReadDeadline error")
 			return nil, nil, nil, 0, err
 		}
 	}
 
-	result = make([]byte, 1024)
+	buff = make([]byte, 1024)
-	len, ra, err = conn.ReadFrom(result)
+	length, ra, err = conn.ReadFrom(buff)
 	if err != nil {
 		// gLog.Println(LevelDEBUG, "ReadFrom error")
 		return nil, nil, nil, 0, err
 	}
 	head = &openP2PHeader{}
-	err = binary.Read(bytes.NewReader(result[:openP2PHeaderSize]), binary.LittleEndian, head)
+	err = binary.Read(bytes.NewReader(buff[:openP2PHeaderSize]), binary.LittleEndian, head)
 	if err != nil {
-		gLog.Println(LvERROR, "parse p2pheader error:", err)
+		gLog.e("parse p2pheader error:%s", err)
 		return nil, nil, nil, 0, err
 	}
+	if head.DataLen > uint32(len(buff)-openP2PHeaderSize) {
+		gLog.e("parse p2pheader error:%d", ErrHeaderDataLen)
+		return nil, nil, nil, 0, ErrHeaderDataLen
+	}
 	return
 }

core/underlay.go

@@ -1,16 +1,68 @@
 package openp2p
 
 import (
+	"io"
+	"net"
 	"time"
 )
 
 type underlay interface {
+	Read([]byte) (int, error)
+	Write([]byte) (int, error)
 	ReadBuffer() (*openP2PHeader, []byte, error)
 	WriteBytes(uint16, uint16, []byte) error
 	WriteBuffer([]byte) error
 	WriteMessage(uint16, uint16, interface{}) error
 	Close() error
+	WLock()
+	WUnlock()
 	SetReadDeadline(t time.Time) error
 	SetWriteDeadline(t time.Time) error
 	Protocol() string
+	RemoteAddr() net.Addr
+}
+
+func DefaultReadBuffer(ul underlay) (*openP2PHeader, []byte, error) {
+	headBuf := make([]byte, openP2PHeaderSize)
+	_, err := io.ReadFull(ul, headBuf)
+	if err != nil {
+		return nil, nil, err
+	}
+	head, err := decodeHeader(headBuf)
+	if err != nil || head.MainType > 16 {
+		gLog.d("DefaultReadBuffer error:%v, %d", err, head.MainType)
+		return nil, nil, err
+	}
+	dataBuf := make([]byte, head.DataLen)
+	_, err = io.ReadFull(ul, dataBuf)
+	return head, dataBuf, err
+}
+
+func DefaultWriteBytes(ul underlay, mainType, subType uint16, data []byte) error {
+	writeBytes := append(encodeHeader(mainType, subType, uint32(len(data))), data...)
+	ul.SetWriteDeadline(time.Now().Add(TunnelHeartbeatTime / 2))
+	ul.WLock()
+	err := writeFull(ul, writeBytes)
+	ul.WUnlock()
+	return err
+}
+
+func DefaultWriteBuffer(ul underlay, data []byte) error {
+	ul.SetWriteDeadline(time.Now().Add(TunnelHeartbeatTime / 2))
+	ul.WLock()
+	err := writeFull(ul, data)
+	ul.WUnlock()
+	return err
+}
+
+func DefaultWriteMessage(ul underlay, mainType uint16, subType uint16, packet interface{}) error {
+	writeBytes, err := newMessage(mainType, subType, packet)
+	if err != nil {
+		return err
+	}
+	ul.SetWriteDeadline(time.Now().Add(TunnelHeartbeatTime / 2))
+	ul.WLock()
+	err = writeFull(ul, writeBytes)
+	ul.WUnlock()
+	return err
 }

core/underlay_kcp.go-

@@ -0,0 +1,95 @@
+package openp2p
+
+import (
+	"fmt"
+	"net"
+	"sync"
+	"time"
+
+	"github.com/xtaci/kcp-go/v5"
+)
+
+type underlayKCP struct {
+	listener *kcp.Listener
+	writeMtx *sync.Mutex
+	*kcp.UDPSession
+}
+
+func (conn *underlayKCP) Protocol() string {
+	return "kcp"
+}
+
+func (conn *underlayKCP) ReadBuffer() (*openP2PHeader, []byte, error) {
+	return DefaultReadBuffer(conn)
+}
+
+func (conn *underlayKCP) WriteBytes(mainType uint16, subType uint16, data []byte) error {
+	return DefaultWriteBytes(conn, mainType, subType, data)
+}
+
+func (conn *underlayKCP) WriteBuffer(data []byte) error {
+	return DefaultWriteBuffer(conn, data)
+}
+
+func (conn *underlayKCP) WriteMessage(mainType uint16, subType uint16, packet interface{}) error {
+	return DefaultWriteMessage(conn, mainType, subType, packet)
+}
+
+func (conn *underlayKCP) Close() error {
+	conn.UDPSession.Close()
+	return nil
+}
+func (conn *underlayKCP) WLock() {
+	conn.writeMtx.Lock()
+}
+func (conn *underlayKCP) WUnlock() {
+	conn.writeMtx.Unlock()
+}
+func (conn *underlayKCP) CloseListener() {
+	if conn.listener != nil {
+		conn.listener.Close()
+	}
+}
+
+func (conn *underlayKCP) Accept() error {
+	kConn, err := conn.listener.AcceptKCP()
+	if err != nil {
+		conn.listener.Close()
+		return err
+	}
+	kConn.SetNoDelay(0, 40, 0, 0)
+	kConn.SetWindowSize(512, 512)
+	kConn.SetWriteBuffer(1024 * 128)
+	kConn.SetReadBuffer(1024 * 128)
+	conn.UDPSession = kConn
+	return nil
+}
+
+func listenKCP(addr string, idleTimeout time.Duration) (*underlayKCP, error) {
+	gLog.d("kcp listen on %s", addr)
+	listener, err := kcp.ListenWithOptions(addr, nil, 0, 0)
+	if err != nil {
+		return nil, fmt.Errorf("quic.ListenAddr error:%s", err)
+	}
+	ul := &underlayKCP{listener: listener, writeMtx: &sync.Mutex{}}
+	err = ul.Accept()
+	if err != nil {
+		ul.CloseListener()
+		return nil, fmt.Errorf("accept KCP error:%s", err)
+	}
+	return ul, nil
+}
+
+func dialKCP(conn *net.UDPConn, remoteAddr *net.UDPAddr, idleTimeout time.Duration) (*underlayKCP, error) {
+	conn.SetDeadline(time.Now().Add(idleTimeout))
+	kConn, err := kcp.NewConn(remoteAddr.String(), nil, 0, 0, conn)
+	if err != nil {
+		return nil, fmt.Errorf("quic.DialContext error:%s", err)
+	}
+	kConn.SetNoDelay(0, 40, 0, 0)
+	kConn.SetWindowSize(512, 512)
+	kConn.SetWriteBuffer(1024 * 128)
+	kConn.SetReadBuffer(1024 * 128)
+	ul := &underlayKCP{nil, &sync.Mutex{}, kConn}
+	return ul, nil
+}

core/underlay_quic.go

@@ -6,19 +6,17 @@ import (
 	"crypto/rsa"
 	"crypto/tls"
 	"crypto/x509"
-	"encoding/json"
 	"encoding/pem"
 	"fmt"
-	"io"
 	"math/big"
 	"net"
 	"sync"
 	"time"
 
-	"github.com/lucas-clemente/quic-go"
+	"github.com/quic-go/quic-go"
 )
 
-//quic.DialContext do not support version 44,disable it
+// quic.DialContext do not support version 44,disable it
 var quicVersion []quic.VersionNumber
 
 type underlayQUIC struct {
@@ -33,53 +31,33 @@ func (conn *underlayQUIC) Protocol() string {
 }
 
 func (conn *underlayQUIC) ReadBuffer() (*openP2PHeader, []byte, error) {
-	headBuf := make([]byte, openP2PHeaderSize)
+	return DefaultReadBuffer(conn)
-	_, err := io.ReadFull(conn, headBuf)
-	if err != nil {
-		return nil, nil, err
-	}
-	head, err := decodeHeader(headBuf)
-	if err != nil {
-		return nil, nil, err
-	}
-	dataBuf := make([]byte, head.DataLen)
-	_, err = io.ReadFull(conn, dataBuf)
-	return head, dataBuf, err
 }
 
 func (conn *underlayQUIC) WriteBytes(mainType uint16, subType uint16, data []byte) error {
-	writeBytes := append(encodeHeader(mainType, subType, uint32(len(data))), data...)
+	return DefaultWriteBytes(conn, mainType, subType, data)
-	conn.writeMtx.Lock()
-	_, err := conn.Write(writeBytes)
-	conn.writeMtx.Unlock()
-	return err
 }
 
 func (conn *underlayQUIC) WriteBuffer(data []byte) error {
-	conn.writeMtx.Lock()
+	return DefaultWriteBuffer(conn, data)
-	_, err := conn.Write(data)
-	conn.writeMtx.Unlock()
-	return err
 }
 
 func (conn *underlayQUIC) WriteMessage(mainType uint16, subType uint16, packet interface{}) error {
-	// TODO: call newMessage
+	return DefaultWriteMessage(conn, mainType, subType, packet)
-	data, err := json.Marshal(packet)
-	if err != nil {
-		return err
-	}
-	writeBytes := append(encodeHeader(mainType, subType, uint32(len(data))), data...)
-	conn.writeMtx.Lock()
-	_, err = conn.Write(writeBytes)
-	conn.writeMtx.Unlock()
-	return err
 }
 
 func (conn *underlayQUIC) Close() error {
 	conn.Stream.CancelRead(1)
 	conn.Connection.CloseWithError(0, "")
+	conn.CloseListener()
 	return nil
 }
+func (conn *underlayQUIC) WLock() {
+	conn.writeMtx.Lock()
+}
+func (conn *underlayQUIC) WUnlock() {
+	conn.writeMtx.Unlock()
+}
 func (conn *underlayQUIC) CloseListener() {
 	if conn.listener != nil {
 		conn.listener.Close()
@@ -87,7 +65,7 @@ func (conn *underlayQUIC) CloseListener() {
 }
 
 func (conn *underlayQUIC) Accept() error {
-	ctx, cancel := context.WithTimeout(context.Background(), time.Second*10)
+	ctx, cancel := context.WithTimeout(context.Background(), UnderlayConnectTimeout)
 	defer cancel()
 	sess, err := conn.listener.Accept(ctx)
 	if err != nil {
@@ -103,22 +81,30 @@ func (conn *underlayQUIC) Accept() error {
 }
 
 func listenQuic(addr string, idleTimeout time.Duration) (*underlayQUIC, error) {
-	gLog.Println(LvDEBUG, "quic listen on ", addr)
+	gLog.d("quic listen on %s", addr)
 	listener, err := quic.ListenAddr(addr, generateTLSConfig(),
 		&quic.Config{Versions: quicVersion, MaxIdleTimeout: idleTimeout, DisablePathMTUDiscovery: true})
 	if err != nil {
 		return nil, fmt.Errorf("quic.ListenAddr error:%s", err)
 	}
-	return &underlayQUIC{listener: listener, writeMtx: &sync.Mutex{}}, nil
+	ul := &underlayQUIC{listener: listener, writeMtx: &sync.Mutex{}}
+	err = ul.Accept()
+	if err != nil {
+		ul.CloseListener()
+		return nil, fmt.Errorf("accept quic error:%s", err)
+	}
+	return ul, nil
 }
 
-func dialQuic(conn *net.UDPConn, remoteAddr *net.UDPAddr, idleTimeout time.Duration) (*underlayQUIC, error) {
+func dialQuic(conn *net.UDPConn, remoteAddr *net.UDPAddr, timeout time.Duration) (*underlayQUIC, error) {
 	tlsConf := &tls.Config{
 		InsecureSkipVerify: true,
 		NextProtos:         []string{"openp2pv1"},
 	}
-	Connection, err := quic.DialContext(context.Background(), conn, remoteAddr, conn.LocalAddr().String(), tlsConf,
+	ctx, cancel := context.WithTimeout(context.Background(), timeout)
-		&quic.Config{Versions: quicVersion, MaxIdleTimeout: idleTimeout, DisablePathMTUDiscovery: true})
+	defer cancel()
+	Connection, err := quic.DialContext(ctx, conn, remoteAddr, conn.LocalAddr().String(), tlsConf,
+		&quic.Config{Versions: quicVersion, MaxIdleTimeout: TunnelIdleTimeout, DisablePathMTUDiscovery: true})
 	if err != nil {
 		return nil, fmt.Errorf("quic.DialContext error:%s", err)
 	}

core/underlay_tcp.go

@@ -1,9 +1,8 @@
 package openp2p
 
 import (
-	"encoding/json"
+	"encoding/binary"
 	"fmt"
-	"io"
 	"net"
 	"sync"
 	"time"
@@ -14,6 +13,7 @@ import (
 type underlayTCP struct {
 	writeMtx *sync.Mutex
 	net.Conn
+	connectTime time.Time
 }
 
 func (conn *underlayTCP) Protocol() string {
@@ -21,88 +21,108 @@ func (conn *underlayTCP) Protocol() string {
 }
 
 func (conn *underlayTCP) ReadBuffer() (*openP2PHeader, []byte, error) {
-	headBuf := make([]byte, openP2PHeaderSize)
+	return DefaultReadBuffer(conn)
-	_, err := io.ReadFull(conn, headBuf)
-	if err != nil {
-		return nil, nil, err
-	}
-	head, err := decodeHeader(headBuf)
-	if err != nil {
-		return nil, nil, err
-	}
-	dataBuf := make([]byte, head.DataLen)
-	_, err = io.ReadFull(conn, dataBuf)
-	return head, dataBuf, err
 }
 
 func (conn *underlayTCP) WriteBytes(mainType uint16, subType uint16, data []byte) error {
-	writeBytes := append(encodeHeader(mainType, subType, uint32(len(data))), data...)
+	return DefaultWriteBytes(conn, mainType, subType, data)
-	conn.writeMtx.Lock()
-	_, err := conn.Write(writeBytes)
-	conn.writeMtx.Unlock()
-	return err
 }
 
 func (conn *underlayTCP) WriteBuffer(data []byte) error {
-	conn.writeMtx.Lock()
+	return DefaultWriteBuffer(conn, data)
-	_, err := conn.Write(data)
-	conn.writeMtx.Unlock()
-	return err
 }
 
 func (conn *underlayTCP) WriteMessage(mainType uint16, subType uint16, packet interface{}) error {
-	// TODO: call newMessage
+	return DefaultWriteMessage(conn, mainType, subType, packet)
-	data, err := json.Marshal(packet)
-	if err != nil {
-		return err
-	}
-	writeBytes := append(encodeHeader(mainType, subType, uint32(len(data))), data...)
-	conn.writeMtx.Lock()
-	_, err = conn.Write(writeBytes)
-	conn.writeMtx.Unlock()
-	return err
 }
 
 func (conn *underlayTCP) Close() error {
 	return conn.Conn.Close()
 }
+func (conn *underlayTCP) WLock() {
+	conn.writeMtx.Lock()
+}
+func (conn *underlayTCP) WUnlock() {
+	conn.writeMtx.Unlock()
+}
 
-func listenTCP(host string, port int, localPort int, mode string) (*underlayTCP, error) {
+func listenTCP(host string, port int, localPort int, mode string, t *P2PTunnel) (underlay, error) {
-	if mode == LinkModeTCPPunch {
+	if mode == LinkModeTCPPunch || mode == LinkModeTCP6 {
-		c, err := reuse.DialTimeout("tcp", fmt.Sprintf("0.0.0.0:%d", localPort), fmt.Sprintf("%s:%d", host, port), SymmetricHandshakeAckTimeout) // TODO: timeout
+		if compareVersion(t.config.peerVersion, SyncServerTimeVersion) < 0 {
+			gLog.d("peer version %s less than %s", t.config.peerVersion, SyncServerTimeVersion)
+		} else {
+			ts := time.Duration(int64(t.punchTs) + GNetwork.dt - time.Now().UnixNano())
+			gLog.d("sleep %d ms", ts/time.Millisecond)
+			time.Sleep(ts)
+		}
+		// gLog.d(" send tcp punch: ", fmt.Sprintf("0.0.0.0:%d", localPort), "-->", fmt.Sprintf("%s:%d", host, port))
+		var c net.Conn
+		var err error
+		if mode == LinkModeTCPPunch {
+			c, err = reuse.DialTimeout("tcp", fmt.Sprintf("0.0.0.0:%d", localPort), fmt.Sprintf("%s:%d", host, port), CheckActiveTimeout)
+		} else {
+			c, err = reuse.DialTimeout("tcp6", fmt.Sprintf("[::]:%d", localPort), fmt.Sprintf("[%s]:%d", t.config.peerIPv6, port), CheckActiveTimeout)
+		}
 		if err != nil {
-			gLog.Println(LvDEBUG, "send tcp punch: ", err)
+			// gLog.d("send tcp punch: ", err)
 			return nil, err
 		}
-		return &underlayTCP{writeMtx: &sync.Mutex{}, Conn: c}, nil
+		utcp := &underlayTCP{writeMtx: &sync.Mutex{}, Conn: c}
+		_, buff, err := utcp.ReadBuffer()
+		if err != nil {
+			return nil, fmt.Errorf("read start msg error:%s", err)
+		}
+		if buff != nil {
+			gLog.d("handshake flag:%s", string(buff))
+		}
+		utcp.WriteBytes(MsgP2P, MsgTunnelHandshakeAck, buff)
+		return utcp, nil
 	}
-	addr, _ := net.ResolveTCPAddr("tcp", fmt.Sprintf("0.0.0.0:%d", localPort))
+	GNetwork.push(t.config.PeerNode, MsgPushUnderlayConnect, nil)
-	l, err := net.ListenTCP("tcp", addr)
+	tid := t.id
-	if err != nil {
+	if compareVersion(t.config.peerVersion, PublicIPVersion) < 0 { // old version
-		return nil, err
+		ipBytes := net.ParseIP(t.config.peerIP).To4()
+		tid = uint64(binary.BigEndian.Uint32(ipBytes))
+		gLog.d("compatible with old client, use ip as key:%d", tid)
 	}
-	l.SetDeadline(time.Now().Add(SymmetricHandshakeAckTimeout))
+	var ul underlay
-	c, err := l.Accept()
+	if v4l != nil {
-	defer l.Close()
+		ul = v4l.getUnderlay(tid)
-	if err != nil {
-		return nil, err
 	}
-	return &underlayTCP{writeMtx: &sync.Mutex{}, Conn: c}, nil
+	if ul == nil {
+		return nil, ErrConnectPublicV4
+	}
+	return ul, nil
 }
 
 func dialTCP(host string, port int, localPort int, mode string) (*underlayTCP, error) {
 	var c net.Conn
 	var err error
-	if mode == LinkModeTCPPunch {
+	network := "tcp"
-		c, err = reuse.DialTimeout("tcp", fmt.Sprintf("0.0.0.0:%d", localPort), fmt.Sprintf("%s:%d", host, port), SymmetricHandshakeAckTimeout)
+	localAddr := fmt.Sprintf("0.0.0.0:%d", localPort)
-	} else {
+	remoteAddr := fmt.Sprintf("%s:%d", host, port)
-		c, err = net.DialTimeout("tcp", fmt.Sprintf("%s:%d", host, port), SymmetricHandshakeAckTimeout)
+	if mode == LinkModeTCP6 { // address need [ip]
+		network = "tcp6"
+		localAddr = fmt.Sprintf("[::]:%d", localPort)
+		remoteAddr = fmt.Sprintf("[%s]:%d", host, port)
+	}
+	if mode == LinkModeTCP4 || mode == LinkModeIntranet { // random port
+		localAddr = fmt.Sprintf("0.0.0.0:%d", 0)
+	}
+	gLog.dev("send tcp punch: %s --> %s", localAddr, remoteAddr)
+
+	c, err = reuse.DialTimeout(network, localAddr, remoteAddr, CheckActiveTimeout)
+	if err != nil {
+		gLog.dev("send tcp punch: %v", err)
 	}
 
 	if err != nil {
-		gLog.Printf(LvERROR, "Dial %s:%d error:%s", host, port, err)
+		gLog.dev("Dial %s:%d error:%s", host, port, err)
 		return nil, err
 	}
-	gLog.Printf(LvDEBUG, "Dial %s:%d OK", host, port)
+	tc := c.(*net.TCPConn)
+	tc.SetKeepAlive(true)
+	tc.SetKeepAlivePeriod(UnderlayTCPKeepalive)
+	gLog.d("Dial %s:%d OK", host, port)
 	return &underlayTCP{writeMtx: &sync.Mutex{}, Conn: c}, nil
 }

core/underlay_tcp6.go

@@ -1,16 +1,13 @@
 package openp2p
 
 import (
-	"encoding/json"
 	"fmt"
-	"io"
 	"net"
 	"sync"
 	"time"
 )
 
 type underlayTCP6 struct {
-	listener net.Listener
 	writeMtx *sync.Mutex
 	net.Conn
 }
@@ -20,60 +17,38 @@ func (conn *underlayTCP6) Protocol() string {
 }
 
 func (conn *underlayTCP6) ReadBuffer() (*openP2PHeader, []byte, error) {
-	headBuf := make([]byte, openP2PHeaderSize)
+	return DefaultReadBuffer(conn)
-	_, err := io.ReadFull(conn, headBuf)
-	if err != nil {
-		return nil, nil, err
-	}
-	head, err := decodeHeader(headBuf)
-	if err != nil {
-		return nil, nil, err
-	}
-	dataBuf := make([]byte, head.DataLen)
-	_, err = io.ReadFull(conn, dataBuf)
-	return head, dataBuf, err
 }
 
 func (conn *underlayTCP6) WriteBytes(mainType uint16, subType uint16, data []byte) error {
-	writeBytes := append(encodeHeader(mainType, subType, uint32(len(data))), data...)
+	return DefaultWriteBytes(conn, mainType, subType, data)
-	conn.writeMtx.Lock()
-	_, err := conn.Write(writeBytes)
-	conn.writeMtx.Unlock()
-	return err
 }
 
 func (conn *underlayTCP6) WriteBuffer(data []byte) error {
-	conn.writeMtx.Lock()
+	return DefaultWriteBuffer(conn, data)
-	_, err := conn.Write(data)
-	conn.writeMtx.Unlock()
-	return err
 }
 
 func (conn *underlayTCP6) WriteMessage(mainType uint16, subType uint16, packet interface{}) error {
-	// TODO: call newMessage
+	return DefaultWriteMessage(conn, mainType, subType, packet)
-	data, err := json.Marshal(packet)
-	if err != nil {
-		return err
-	}
-	writeBytes := append(encodeHeader(mainType, subType, uint32(len(data))), data...)
-	conn.writeMtx.Lock()
-	_, err = conn.Write(writeBytes)
-	conn.writeMtx.Unlock()
-	return err
 }
 
 func (conn *underlayTCP6) Close() error {
 	return conn.Conn.Close()
 }
-
+func (conn *underlayTCP6) WLock() {
-func listenTCP6(port int, idleTimeout time.Duration) (*underlayTCP6, error) {
+	conn.writeMtx.Lock()
+}
+func (conn *underlayTCP6) WUnlock() {
+	conn.writeMtx.Unlock()
+}
+func listenTCP6(port int, timeout time.Duration) (*underlayTCP6, error) {
 	addr, _ := net.ResolveTCPAddr("tcp6", fmt.Sprintf("[::]:%d", port))
 	l, err := net.ListenTCP("tcp6", addr)
 	if err != nil {
 		return nil, err
 	}
 	defer l.Close()
-	l.SetDeadline(time.Now().Add(SymmetricHandshakeAckTimeout))
+	l.SetDeadline(time.Now().Add(timeout))
 	c, err := l.Accept()
 	defer l.Close()
 	if err != nil {
@@ -83,7 +58,7 @@ func listenTCP6(port int, idleTimeout time.Duration) (*underlayTCP6, error) {
 }
 
 func dialTCP6(host string, port int) (*underlayTCP6, error) {
-	c, err := net.DialTimeout("tcp6", fmt.Sprintf("[%s]:%d", host, port), SymmetricHandshakeAckTimeout)
+	c, err := net.DialTimeout("tcp6", fmt.Sprintf("[%s]:%d", host, port), UnderlayConnectTimeout)
 	if err != nil {
 		gLog.Printf(LvERROR, "Dial %s:%d error:%s", host, port, err)
 		return nil, err

core/underlay_tcp_test.go

@@ -0,0 +1,17 @@
+package openp2p
+
+import (
+	"testing"
+)
+
+func TestDialTCP(t *testing.T) {
+	InitForUnitTest(LvDEBUG)
+	// ul, err := dialTCP("[240e:3b1:6f6:d14:1c0b:9605:554d:351c]", 3389, 0, LinkModeTCP6)
+	// if err != nil || ul == nil {
+	// 	t.Error("dialTCP error:", err)
+	// }
+	ul, err := dialTCP("192.168.3.9", 3389, 0, LinkModeTCP6)
+	if err != nil || ul == nil {
+		t.Error("dialTCP error:", err)
+	}
+}

core/update.go

@@ -1,193 +1,260 @@
-package openp2p
+package openp2p
-
+
-import (
+import (
-	"archive/tar"
+	"archive/tar"
-	"archive/zip"
+	"archive/zip"
-	"compress/gzip"
+	"compress/gzip"
-	"crypto/tls"
+	"crypto/tls"
-	"encoding/json"
+	"crypto/x509"
-	"fmt"
+	"encoding/json"
-	"io"
+	"fmt"
-	"io/ioutil"
+	"io"
-	"net/http"
+	"net/http"
-	"os"
+	"net/url"
-	"path/filepath"
+	"os"
-	"runtime"
+	"path/filepath"
-	"time"
+	"runtime"
-)
+	"time"
-
+)
-func update(host string, port int) {
+
-	gLog.Println(LvINFO, "update start")
+func update(host string, port int) error {
-	defer gLog.Println(LvINFO, "update end")
+	gLog.i("update start")
-	c := http.Client{
+	defer gLog.i("update end")
-		Transport: &http.Transport{
+	caCertPool, err := x509.SystemCertPool()
-			TLSClientConfig: &tls.Config{InsecureSkipVerify: true},
+	if err != nil {
-		},
+		gLog.e("Failed to load system root CAs:%s", err)
-		Timeout: time.Second * 30,
+		caCertPool = x509.NewCertPool()
-	}
+	}
-	goos := runtime.GOOS
+	caCertPool.AppendCertsFromPEM([]byte(rootCA))
-	goarch := runtime.GOARCH
+	caCertPool.AppendCertsFromPEM([]byte(rootEdgeCA))
-	rsp, err := c.Get(fmt.Sprintf("https://%s:%d/api/v1/update?fromver=%s&os=%s&arch=%s", host, port, OpenP2PVersion, goos, goarch))
+	caCertPool.AppendCertsFromPEM([]byte(ISRGRootX1))
-	if err != nil {
+
-		gLog.Println(LvERROR, "update:query update list failed:", err)
+	c := http.Client{
-		return
+		Transport: &http.Transport{
-	}
+			TLSClientConfig: &tls.Config{RootCAs: caCertPool,
-	defer rsp.Body.Close()
+				InsecureSkipVerify: gConf.TLSInsecureSkipVerify},
-	if rsp.StatusCode != http.StatusOK {
+		},
-		gLog.Println(LvERROR, "get update info error:", rsp.Status)
+		Timeout: time.Second * 30,
-		return
+	}
-	}
+	goos := runtime.GOOS
-	rspBuf, err := ioutil.ReadAll(rsp.Body)
+	goarch := runtime.GOARCH
-	if err != nil {
+	rsp, err := c.Get(fmt.Sprintf("https://%s:%d/api/v1/update?fromver=%s&os=%s&arch=%s&user=%s&node=%s", host, port, OpenP2PVersion, goos, goarch, url.QueryEscape(gConf.Network.User), url.QueryEscape(gConf.Network.Node)))
-		gLog.Println(LvERROR, "update:read update list failed:", err)
+	if err != nil {
-		return
+		gLog.e("update:query update list failed:%s", err)
-	}
+		return err
-	updateInfo := UpdateInfo{}
+	}
-	err = json.Unmarshal(rspBuf, &updateInfo)
+	defer rsp.Body.Close()
-	if err != nil {
+	if rsp.StatusCode != http.StatusOK {
-		gLog.Println(LvERROR, rspBuf, " update info decode error:", err)
+		gLog.e("get update info error:%s", rsp.Status)
-		return
+		return err
-	}
+	}
-	if updateInfo.Error != 0 {
+	rspBuf, err := io.ReadAll(rsp.Body)
-		gLog.Println(LvERROR, "update error:", updateInfo.Error, updateInfo.ErrorDetail)
+	if err != nil {
-		return
+		gLog.e("update:read update list failed:%s", err)
-	}
+		return err
-	err = updateFile(updateInfo.Url, "", "openp2p")
+	}
-	if err != nil {
+	updateInfo := UpdateInfo{}
-		gLog.Println(LvERROR, "update: download failed:", err)
+	if err = json.Unmarshal(rspBuf, &updateInfo); err != nil {
-		return
+		gLog.e("%s update info decode error:%s", string(rspBuf), err)
-	}
+		return err
-}
+	}
-
+	if updateInfo.Error != 0 {
-// todo rollback on error
+		gLog.e("update error:%d,%s", updateInfo.Error, updateInfo.ErrorDetail)
-func updateFile(url string, checksum string, dst string) error {
+		return err
-	gLog.Println(LvINFO, "download ", url)
+	}
-	tmpFile := filepath.Dir(os.Args[0]) + "/openp2p.tmp"
+	err = updateFile(updateInfo.Url, "", "openp2p")
-	output, err := os.OpenFile(tmpFile, os.O_CREATE|os.O_WRONLY|os.O_TRUNC, 0776)
+	if err != nil {
-	if err != nil {
+		gLog.e("update: download failed:%s, retry...", err)
-		gLog.Printf(LvERROR, "OpenFile %s error:%s", tmpFile, err)
+		err = updateFile(updateInfo.Url2, "", "openp2p")
-		return err
+		if err != nil {
-	}
+			gLog.e("update: download failed:%s", err)
-	tr := &http.Transport{
+			return err
-		TLSClientConfig: &tls.Config{InsecureSkipVerify: true},
+		}
-	}
+	}
-	client := &http.Client{Transport: tr}
+	return nil
-	response, err := client.Get(url)
+}
-	if err != nil {
+
-		gLog.Printf(LvERROR, "download url %s error:%s", url, err)
+func downloadFile(url string, checksum string, dstFile string) error {
-		output.Close()
+	output, err := os.OpenFile(dstFile, os.O_CREATE|os.O_WRONLY|os.O_TRUNC, 0776)
-		return err
+	if err != nil {
-	}
+		gLog.e("OpenFile %s error:%s", dstFile, err)
-	defer response.Body.Close()
+		return err
-	n, err := io.Copy(output, response.Body)
+	}
-	if err != nil {
+	caCertPool, err := x509.SystemCertPool()
-		gLog.Printf(LvERROR, "io.Copy error:%s", err)
+	if err != nil {
-		output.Close()
+		gLog.e("Failed to load system root CAs:%s", err)
-		return err
+		caCertPool = x509.NewCertPool()
-	}
+	}
-	output.Sync()
+	caCertPool.AppendCertsFromPEM([]byte(rootCA))
-	output.Close()
+	caCertPool.AppendCertsFromPEM([]byte(rootEdgeCA))
-	gLog.Println(LvINFO, "download ", url, " ok")
+	caCertPool.AppendCertsFromPEM([]byte(ISRGRootX1))
-	gLog.Printf(LvINFO, "size: %d bytes", n)
+	tr := &http.Transport{
-
+		TLSClientConfig: &tls.Config{
-	err = os.Rename(os.Args[0], os.Args[0]+"0")
+			RootCAs:            caCertPool,
-	if err != nil && os.IsExist(err) {
+			InsecureSkipVerify: gConf.TLSInsecureSkipVerify},
-		gLog.Printf(LvINFO, " rename %s error:%s", os.Args[0], err)
+	}
-	}
+	client := &http.Client{Transport: tr,
-	// extract
+		Timeout: 60 * time.Second}
-	gLog.Println(LvINFO, "extract files")
+	response, err := client.Get(url)
-	err = extract(filepath.Dir(os.Args[0]), tmpFile)
+	if err != nil {
-	if err != nil {
+		gLog.e("download url %s error:%s", url, err)
-		gLog.Printf(LvERROR, "extract error:%s. revert rename", err)
+		output.Close()
-		os.Rename(os.Args[0]+"0", os.Args[0])
+		return err
-		return err
+	}
-	}
+	defer response.Body.Close()
-	os.Remove(tmpFile)
+	n, err := io.Copy(output, response.Body)
-	return nil
+	if err != nil {
-}
+		gLog.e("io.Copy error:%s", err)
-
+		output.Close()
-func extract(dst, src string) (err error) {
+		return err
-	if runtime.GOOS == "windows" {
+	}
-		return unzip(dst, src)
+	output.Sync()
-	} else {
+	output.Close()
-		return extractTgz(dst, src)
+	gLog.i("download %s ok", url)
-	}
+	gLog.i("size: %d bytes", n)
-}
+	return nil
-
+}
-func unzip(dst, src string) (err error) {
+
-	archive, err := zip.OpenReader(src)
+func updateFile(url string, checksum string, dst string) error {
-	if err != nil {
+	gLog.i("download %s", url)
-		return err
+	tempDir := os.TempDir()
-	}
+	tmpFile := filepath.Join(tempDir, "openp2p.tmp")
-	defer archive.Close()
+	err := downloadFile(url, checksum, tmpFile)
-
+	if err != nil {
-	for _, f := range archive.File {
+		return err
-		filePath := filepath.Join(dst, f.Name)
+	}
-		fmt.Println("unzipping file ", filePath)
+	backupBase := filepath.Base(os.Args[0])
-		if f.FileInfo().IsDir() {
+	var backupFile string
-			fmt.Println("creating directory...")
+	if runtime.GOOS == "windows" {
-			os.MkdirAll(filePath, os.ModePerm)
+		backupFile = filepath.Join(tempDir, backupBase+"0")
-			continue
+	} else {
-		}
+		backupFile = os.Args[0] + "0" // linux can not mv running executable to /tmp, because they are different volumns
-		if err := os.MkdirAll(filepath.Dir(filePath), os.ModePerm); err != nil {
+	}
-			return err
+	gLog.i("backup file %s --> %s", os.Args[0], backupFile)
-		}
+	err = moveFile(os.Args[0], backupFile)
-		dstFile, err := os.OpenFile(filePath, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, f.Mode())
+	if err != nil {
-		if err != nil {
+		if runtime.GOOS == "windows" {
-			return err
+			backupFile = filepath.Join(tempDir, backupBase+"1")
-		}
+		} else {
-		fileInArchive, err := f.Open()
+			backupFile = os.Args[0] + "1" // 1st update will mv deamon process to 0, 2nd update mv to 0 will failed, mv to 1
-		if err != nil {
+		}
-			return err
+		gLog.i("backup file %s --> %s", os.Args[0], backupFile)
-		}
+		err = moveFile(os.Args[0], backupFile)
-		if _, err := io.Copy(dstFile, fileInArchive); err != nil {
+		if err != nil {
-			return err
+			gLog.e(" rename %s error:%s", os.Args[0], err)
-		}
+			return err
-		dstFile.Close()
+		}
-		fileInArchive.Close()
+	}
-	}
+	// extract
-	return nil
+	gLog.i("extract files")
-}
+	err = extract(filepath.Dir(os.Args[0]), tmpFile)
-
+	if err != nil {
-func extractTgz(dst, src string) error {
+		gLog.e("extract error:%s. revert rename", err)
-	gzipStream, err := os.Open(src)
+		moveFile(backupFile, os.Args[0])
-	if err != nil {
+		return err
-		return err
+	}
-	}
+	os.Remove(tmpFile)
-	uncompressedStream, err := gzip.NewReader(gzipStream)
+	return nil
-	if err != nil {
+}
-		return err
+
-	}
+func extract(dst, src string) (err error) {
-	tarReader := tar.NewReader(uncompressedStream)
+	if runtime.GOOS == "windows" {
-	for {
+		return unzip(dst, src)
-		header, err := tarReader.Next()
+	} else {
-		if err == io.EOF {
+		return extractTgz(dst, src)
-			break
+	}
-		}
+}
-		if err != nil {
+
-			return err
+func unzip(dst, src string) (err error) {
-		}
+	archive, err := zip.OpenReader(src)
-		switch header.Typeflag {
+	if err != nil {
-		case tar.TypeDir:
+		return err
-			if err := os.Mkdir(header.Name, 0755); err != nil {
+	}
-				return err
+	defer archive.Close()
-			}
+
-		case tar.TypeReg:
+	for _, f := range archive.File {
-			filePath := filepath.Join(dst, header.Name)
+		filePath := filepath.Join(dst, f.Name)
-			outFile, err := os.OpenFile(filePath, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, os.FileMode(header.Mode))
+		fmt.Println("unzipping file ", filePath)
-			if err != nil {
+		if f.FileInfo().IsDir() {
-				return err
+			fmt.Println("creating directory...")
-			}
+			os.MkdirAll(filePath, os.ModePerm)
-			if err != nil {
+			continue
-				return err
+		}
-			}
+		if err := os.MkdirAll(filepath.Dir(filePath), os.ModePerm); err != nil {
-			defer outFile.Close()
+			return err
-			if _, err := io.Copy(outFile, tarReader); err != nil {
+		}
-				return err
+		dstFile, err := os.OpenFile(filePath, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, f.Mode())
-			}
+		if err != nil {
-		default:
+			return err
-			return err
+		}
-		}
+		fileInArchive, err := f.Open()
-	}
+		if err != nil {
-	return nil
+			return err
-}
+		}
+		if _, err := io.Copy(dstFile, fileInArchive); err != nil {
+			return err
+		}
+		dstFile.Close()
+		fileInArchive.Close()
+	}
+	return nil
+}
+
+func extractTgz(dst, src string) error {
+	gzipStream, err := os.Open(src)
+	if err != nil {
+		return err
+	}
+	uncompressedStream, err := gzip.NewReader(gzipStream)
+	if err != nil {
+		return err
+	}
+	tarReader := tar.NewReader(uncompressedStream)
+	for {
+		header, err := tarReader.Next()
+		if err == io.EOF {
+			break
+		}
+		if err != nil {
+			return err
+		}
+		switch header.Typeflag {
+		case tar.TypeDir:
+			if err := os.Mkdir(header.Name, 0755); err != nil {
+				return err
+			}
+		case tar.TypeReg:
+			filePath := filepath.Join(dst, header.Name)
+			outFile, err := os.OpenFile(filePath, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, os.FileMode(header.Mode))
+			if err != nil {
+				return err
+			}
+			defer outFile.Close()
+			if _, err := io.Copy(outFile, tarReader); err != nil {
+				return err
+			}
+		default:
+			return err
+		}
+	}
+	return nil
+}
+
+func cleanTempFiles() {
+	tempDir := os.TempDir()
+	backupBase := filepath.Base(os.Args[0])
+	for i := 0; i < 2; i++ {
+		tmpFile := fmt.Sprintf("%s%d", os.Args[0], i)
+		if _, err := os.Stat(tmpFile); err == nil {
+			if err := os.Remove(tmpFile); err != nil {
+				gLog.d(" remove %s error:%s", tmpFile, err)
+			}
+		}
+		tmpFile = fmt.Sprintf("%s%s%d", tempDir, backupBase, i)
+		if _, err := os.Stat(tmpFile); err == nil {
+			if err := os.Remove(tmpFile); err != nil {
+				gLog.d(" remove %s error:%s", tmpFile, err)
+			}
+		}
+	}
+}

core/upnp.go

@@ -5,6 +5,7 @@ package openp2p
 
 import (
 	"bytes"
+	"crypto/tls"
 	"encoding/xml"
 	"errors"
 	"fmt"
@@ -62,13 +63,15 @@ func Discover() (nat NAT, err error) {
 			return
 		}
 		var n int
+		socket.SetDeadline(time.Now().Add(3 * time.Second))
 		_, _, err = socket.ReadFromUDP(answerBytes)
 		if err != nil {
-			gLog.Println(LvDEBUG, "UPNP discover error:", err)
+			gLog.d("UPNP discover error:%s", err)
 			return
 		}
 
 		for {
+			socket.SetDeadline(time.Now().Add(3 * time.Second))
 			n, _, err = socket.ReadFromUDP(answerBytes)
 			if err != nil {
 				break
@@ -181,7 +184,12 @@ func localIPv4() string { // TODO: multi nic will wrong
 }
 
 func getServiceURL(rootURL string) (url, urnDomain string, err error) {
-	r, err := http.Get(rootURL)
+	client := &http.Client{
+		Transport: &http.Transport{
+			TLSClientConfig: &tls.Config{InsecureSkipVerify: true},
+		},
+		Timeout: time.Second * 3}
+	r, err := client.Get(rootURL)
 	if err != nil {
 		return
 	}
@@ -260,7 +268,11 @@ func soapRequest(url, function, message, domain string) (r *http.Response, err e
 
 	// log.Stderr("soapRequest ", req)
 
-	r, err = http.DefaultClient.Do(req)
+	client := &http.Client{
+		Timeout: 3 * time.Second,
+	}
+
+	r, err = client.Do(req)
 	if err != nil {
 		return nil, err
 	}
@@ -352,11 +364,6 @@ func (n *upnpNAT) AddPortMapping(protocol string, externalPort, internalPort int
 		return
 	}
 
-	// TODO: check response to see if the port was forwarded
-	// log.Println(message, response)
-	// JAE:
-	// body, err := ioutil.ReadAll(response.Body)
-	// fmt.Println(string(body), err)
 	mappedExternalPort = externalPort
 	_ = response
 	return
@@ -378,8 +385,6 @@ func (n *upnpNAT) DeletePortMapping(protocol string, externalPort, internalPort
 		return
 	}
 
-	// TODO: check response to see if the port was deleted
-	// log.Println(message, response)
 	_ = response
 	return
 }

core/util_darwin.go

@@ -5,9 +5,12 @@ import (
 	"syscall"
 )
 
-const (
+var (
 	defaultInstallPath = "/usr/local/openp2p"
-	defaultBinName     = "openp2p"
+)
+
+const (
+	defaultBinName = "openp2p"
 )
 
 func getOsName() (osName string) {
@@ -21,7 +24,7 @@ func setRLimit() error {
 	if err := syscall.Getrlimit(syscall.RLIMIT_NOFILE, &limit); err != nil {
 		return err
 	}
-	limit.Cur = 10240
+	limit.Cur = 65536
 	if err := syscall.Setrlimit(syscall.RLIMIT_NOFILE, &limit); err != nil {
 		return err
 	}

core/util_freebsd.go

@@ -0,0 +1,71 @@
+package openp2p
+
+import (
+	"bufio"
+	"bytes"
+	"io/ioutil"
+	"os"
+	"strings"
+	"syscall"
+)
+
+var (
+	defaultInstallPath = "/usr/local/openp2p"
+)
+
+const (
+	defaultBinName = "openp2p"
+)
+
+func getOsName() (osName string) {
+	var sysnamePath string
+	sysnamePath = "/etc/redhat-release"
+	_, err := os.Stat(sysnamePath)
+	if err != nil && os.IsNotExist(err) {
+		str := "PRETTY_NAME="
+		f, err := os.Open("/etc/os-release")
+		if err == nil {
+			buf := bufio.NewReader(f)
+			for {
+				line, err := buf.ReadString('\n')
+				if err == nil {
+					line = strings.TrimSpace(line)
+					pos := strings.Count(line, str)
+					if pos > 0 {
+						len1 := len([]rune(str)) + 1
+						rs := []rune(line)
+						osName = string(rs[len1 : (len(rs))-1])
+						break
+					}
+				} else {
+					break
+				}
+			}
+		}
+	} else {
+		buff, err := ioutil.ReadFile(sysnamePath)
+		if err == nil {
+			osName = string(bytes.TrimSpace(buff))
+		}
+	}
+	if osName == "" {
+		osName = "FreeBSD"
+	}
+	return
+}
+
+func setRLimit() error {
+	var limit syscall.Rlimit
+	if err := syscall.Getrlimit(syscall.RLIMIT_NOFILE, &limit); err != nil {
+		return err
+	}
+	limit.Max = 65536
+	limit.Cur = limit.Max
+	if err := syscall.Setrlimit(syscall.RLIMIT_NOFILE, &limit); err != nil {
+		return err
+	}
+	return nil
+}
+
+func setFirewall() {
+}

core/util_linux.go

@@ -10,9 +10,12 @@ import (
 	"syscall"
 )
 
-const (
+var (
 	defaultInstallPath = "/usr/local/openp2p"
-	defaultBinName     = "openp2p"
+)
+
+const (
+	defaultBinName = "openp2p"
 )
 
 func getOsName() (osName string) {
@@ -64,7 +67,7 @@ func setRLimit() error {
 	if err := syscall.Getrlimit(syscall.RLIMIT_NOFILE, &limit); err != nil {
 		return err
 	}
-	limit.Max = 1024 * 1024
+	limit.Max = 65536
 	limit.Cur = limit.Max
 	if err := syscall.Setrlimit(syscall.RLIMIT_NOFILE, &limit); err != nil {
 		return err

core/util_windows.go

@@ -5,14 +5,18 @@ import (
 	"os"
 	"os/exec"
 	"path/filepath"
+	"strconv"
 	"strings"
 
 	"golang.org/x/sys/windows/registry"
 )
 
-const (
+var (
 	defaultInstallPath = "C:\\Program Files\\OpenP2P"
-	defaultBinName     = "openp2p.exe"
+)
+
+const (
+	defaultBinName = "openp2p.exe"
 )
 
 func getOsName() (osName string) {
@@ -23,6 +27,17 @@ func getOsName() (osName string) {
 	defer k.Close()
 	pn, _, err := k.GetStringValue("ProductName")
 	if err == nil {
+		currentBuild, _, err := k.GetStringValue("CurrentBuild")
+		if err != nil {
+			return
+		}
+		buildNumber, err := strconv.Atoi(currentBuild)
+		if err != nil {
+			return
+		}
+		if buildNumber >= 22000 {
+			pn = strings.Replace(pn, "Windows 10", "Windows 11", 1)
+		}
 		osName = pn
 	}
 	return
@@ -35,7 +50,7 @@ func setRLimit() error {
 func setFirewall() {
 	fullPath, err := filepath.Abs(os.Args[0])
 	if err != nil {
-		gLog.Println(LvERROR, "add firewall error:", err)
+		gLog.e("add firewall error:%s", err)
 		return
 	}
 	isXP := false
@@ -45,9 +60,9 @@ func setFirewall() {
 	}
 	if isXP {
 		exec.Command("cmd.exe", `/c`, fmt.Sprintf(`netsh firewall del allowedprogram "%s"`, fullPath)).Run()
-		exec.Command("cmd.exe", `/c`, fmt.Sprintf(`netsh firewall add allowedprogram "%s" "%s" ENABLE`, ProducnName, fullPath)).Run()
+		exec.Command("cmd.exe", `/c`, fmt.Sprintf(`netsh firewall add allowedprogram "%s" "%s" ENABLE`, ProductName, fullPath)).Run()
 	} else { // win7 or later
-		exec.Command("cmd.exe", `/c`, fmt.Sprintf(`netsh advfirewall firewall del rule name="%s"`, ProducnName)).Run()
+		exec.Command("cmd.exe", `/c`, fmt.Sprintf(`netsh advfirewall firewall del rule name="%s"`, ProductName)).Run()
-		exec.Command("cmd.exe", `/c`, fmt.Sprintf(`netsh advfirewall firewall add rule name="%s" dir=in action=allow program="%s" enable=yes`, ProducnName, fullPath)).Run()
+		exec.Command("cmd.exe", `/c`, fmt.Sprintf(`netsh advfirewall firewall add rule name="%s" dir=in action=allow program="%s" enable=yes`, ProductName, fullPath)).Run()
 	}
 }

core/v4listener.go

@@ -0,0 +1,157 @@
+package openp2p
+
+import (
+	"context"
+	"encoding/binary"
+	"fmt"
+	"net"
+	"sync"
+	"time"
+
+	"github.com/quic-go/quic-go"
+)
+
+type v4Listener struct {
+	conns       sync.Map
+	port        int
+	acceptCh    chan bool
+	running     bool
+	tcpListener *net.TCPListener
+	udpListener quic.Listener
+	wg          sync.WaitGroup
+}
+
+func (vl *v4Listener) start() {
+	vl.running = true
+	v4l.acceptCh = make(chan bool, 500)
+	vl.wg.Add(1)
+	go func() {
+		defer vl.wg.Done()
+		for vl.running {
+			vl.listenTCP()
+			time.Sleep(UnderlayTCPConnectTimeout)
+		}
+	}()
+	vl.wg.Add(1)
+	go func() {
+		defer vl.wg.Done()
+		for vl.running {
+			vl.listenUDP()
+			time.Sleep(UnderlayTCPConnectTimeout)
+		}
+	}()
+}
+
+func (vl *v4Listener) stop() {
+	vl.running = false
+	if vl.tcpListener != nil {
+		vl.tcpListener.Close()
+	}
+	if vl.udpListener != nil {
+		vl.udpListener.Close()
+	}
+	vl.wg.Wait()
+}
+
+func (vl *v4Listener) listenTCP() error {
+	gLog.d("v4Listener listenTCP %d start", vl.port)
+	defer gLog.d("v4Listener listenTCP %d end", vl.port)
+	addr, _ := net.ResolveTCPAddr("tcp", fmt.Sprintf("0.0.0.0:%d", vl.port)) // system will auto listen both v4 and v6
+	var err error
+	vl.tcpListener, err = net.ListenTCP("tcp", addr)
+	if err != nil {
+		gLog.e("v4Listener listen %d error:", vl.port, err)
+		return err
+	}
+	defer vl.tcpListener.Close()
+	for {
+		c, err := vl.tcpListener.Accept()
+		if err != nil {
+			break
+		}
+		utcp := &underlayTCP{writeMtx: &sync.Mutex{}, Conn: c, connectTime: time.Now()}
+		go vl.handleConnection(utcp)
+	}
+	vl.tcpListener = nil
+	return nil
+}
+
+func (vl *v4Listener) listenUDP() error {
+	gLog.d("v4Listener listenUDP %d start", vl.port)
+	defer gLog.d("v4Listener listenUDP %d end", vl.port)
+	var err error
+	vl.udpListener, err = quic.ListenAddr(fmt.Sprintf("0.0.0.0:%d", vl.port), generateTLSConfig(),
+		&quic.Config{Versions: quicVersion, MaxIdleTimeout: TunnelIdleTimeout, DisablePathMTUDiscovery: true})
+	if err != nil {
+		return err
+	}
+	ctx, cancel := context.WithTimeout(context.Background(), UnderlayConnectTimeout)
+	defer cancel()
+	defer vl.udpListener.Close()
+	for {
+		sess, err := vl.udpListener.Accept(context.Background())
+		if err != nil {
+			break
+		}
+		stream, err := sess.AcceptStream(ctx)
+		if err != nil {
+			break
+		}
+		ul := &underlayQUIC{writeMtx: &sync.Mutex{}, Stream: stream, Connection: sess}
+		go vl.handleConnection(ul)
+	}
+	vl.udpListener = nil
+	return err
+}
+
+func (vl *v4Listener) handleConnection(ul underlay) {
+	gLog.d("v4Listener accept connection: %s", ul.RemoteAddr().String())
+	ul.SetReadDeadline(time.Now().Add(UnderlayTCPConnectTimeout))
+	_, buff, err := ul.ReadBuffer()
+	if err != nil || buff == nil {
+		gLog.e("v4Listener read MsgTunnelHandshake error:%s", err)
+	}
+	ul.WriteBytes(MsgP2P, MsgTunnelHandshakeAck, buff)
+	var tid uint64
+	if string(buff) == "OpenP2P,hello" { // old client
+		// save remoteIP as key
+		remoteAddr := ul.RemoteAddr().(*net.TCPAddr).IP
+		ipBytes := remoteAddr.To4()
+		tid = uint64(binary.BigEndian.Uint32(ipBytes)) // bytes not enough for uint64
+		gLog.d("hello %s", string(buff))
+	} else {
+		if len(buff) < 8 {
+			return
+		}
+		tid = binary.LittleEndian.Uint64(buff[:8])
+		gLog.d("hello %d", tid)
+	}
+	// clear timeout connections
+	vl.conns.Range(func(idx, i interface{}) bool {
+		if ut, ok := i.(*underlayTCP); ok {
+			if ut.connectTime.Before(time.Now().Add(-UnderlayTCPConnectTimeout)) {
+				vl.conns.Delete(idx)
+			}
+		}
+		return true
+	})
+	vl.conns.Store(tid, ul)
+	select {
+	case vl.acceptCh <- true:
+	default:
+		gLog.e("msgQueue full, drop it")
+	}
+}
+
+func (vl *v4Listener) getUnderlay(tid uint64) underlay {
+	for i := 0; i < 100; i++ {
+		select {
+		case <-time.After(time.Millisecond * 50):
+		case <-vl.acceptCh:
+		}
+		if u, ok := vl.conns.LoadAndDelete(tid); ok {
+			return u.(underlay)
+		}
+	}
+	return nil
+}

docker/Dockerfile

@@ -0,0 +1,14 @@
+FROM  alpine:3.18.2
+
+# Replace the default Alpine repositories with Aliyun mirrors
+RUN sed -i 's/dl-cdn.alpinelinux.org/mirrors.aliyun.com/g' /etc/apk/repositories && \
+    apk add --no-cache ca-certificates iptables && \
+    rm -rf /tmp/* /var/tmp/* /var/cache/apk/* /var/cache/distfiles/*
+
+COPY get-client.sh /
+ARG VERSION
+LABEL version=${VERSION}
+# ARG DOCKER_VER="latest"
+RUN echo $TARGETPLATFORM && chmod +x /get-client.sh && ./get-client.sh
+
+ENTRYPOINT ["/usr/local/openp2p/openp2p"]

docker/get-client.sh

@@ -0,0 +1,46 @@
+#!/bin/sh
+
+
+echo "Building version:${VERSION}"
+echo "Running on platform: $TARGETPLATFORM"
+# TARGETPLATFORM=$(echo $TARGETPLATFORM | tr ',' '/')
+echo "Running on platform: $TARGETPLATFORM"
+sysType="linux-amd64"
+		archType=$(uname -m)
+		if [[ $archType == aarch64 ]] ;
+		then
+		    sysType="linux-arm64"
+		elif  [[ $archType == arm* ]] ;
+		then
+			sysType="linux-arm"
+		elif  [[ $archType == i*86 ]] ;
+		then
+			sysType="linux-386"
+		elif  [[ $archType == mips ]] ;
+		then
+			sysType="linux-mipsle"
+			ls /lib |grep mipsel
+			if [[ $? -ne 0 ]]; then
+				# mipsel not found, it's mipseb
+				sysType="linux-mipsbe"
+			fi
+		fi
+url="https://console.openpxp.com/download/v1/${VERSION}/openp2p-${VERSION}.$sysType.tar.gz"
+echo "download $url start"
+
+if [ -f /usr/bin/curl ]; then
+	curl -k -o openp2p.tar.gz $url
+else
+  wget --no-check-certificate -O openp2p.tar.gz $url
+fi
+if [ $? -ne 0 ]; then
+	echo "download error $?"
+    exit 9
+fi
+echo "download ok"
+mkdir -p /usr/local/openp2p/
+tar -xzvf openp2p.tar.gz -C /usr/local/openp2p/
+chmod +x /usr/local/openp2p/openp2p
+pwd
+ls -l
+exit 0

example/dll/dll.cpp

@@ -0,0 +1,14 @@
+#include <iostream>
+#include <windows.h>
+
+using namespace std;
+typedef void (*pRun)(const char *);
+
+int main(int argc, char *argv[])
+{
+	HMODULE dll = LoadLibraryA("openp2p.dll");
+	pRun run = (pRun)GetProcAddress(dll, "RunCmd");
+	run("-node 5800-debug2 -token YOUR-TOKEN");
+	FreeLibrary(dll);
+	return 0;
+}

example/echo/echo-client.go

@@ -0,0 +1,42 @@
+package main
+
+import (
+	"fmt"
+	op2p "openp2p/core"
+	"time"
+)
+
+func main() {
+	op2p.Run()
+	for i := 0; i < 10; i++ {
+		go echoClient("5800-debug")
+	}
+	echoClient("5800-debug")
+}
+
+func echoClient(peerNode string) {
+	sendDatalen := op2p.ReadBuffLen
+	sendBuff := make([]byte, sendDatalen)
+	for i := 0; i < len(sendBuff); i++ {
+		sendBuff[i] = byte('A' + i/100)
+	}
+	// peerNode = "YOUR-PEER-NODE-NAME"
+	if err := op2p.GNetwork.ConnectNode(peerNode); err != nil {
+		fmt.Println("connect error:", err)
+		return
+	}
+	for i := 0; ; i++ {
+		sendBuff[1] = 'A' + byte(i%26)
+		if err := op2p.GNetwork.WriteNode(op2p.NodeNameToID(peerNode), sendBuff[:sendDatalen]); err != nil {
+			fmt.Println("write error:", err)
+			break
+		}
+		nd := op2p.GNetwork.ReadNode(time.Second * 10)
+		if nd == nil {
+			fmt.Printf("waiting for node data\n")
+			time.Sleep(time.Second * 10)
+			continue
+		}
+		fmt.Printf("read len=%d data=%s\n", len(nd), nd[:16]) // only print 16 bytes
+	}
+}

example/echo/echo-server.go

@@ -0,0 +1,32 @@
+package main
+
+import (
+	"fmt"
+	op2p "openp2p/core"
+	"time"
+)
+
+func main() {
+	op2p.Run()
+	echoServer()
+	forever := make(chan bool)
+	<-forever
+}
+
+func echoServer() {
+	// peerID := fmt.Sprintf("%d", core.NodeNameToID(peerNode))
+	for {
+		nd := op2p.GNetwork.ReadNode(time.Second * 10)
+		if nd == nil {
+			fmt.Printf("waiting for node data\n")
+			// time.Sleep(time.Second * 10)
+			continue
+		}
+		// fmt.Printf("read %s len=%d data=%s\n", nd.Node, len(nd.Data), nd.Data[:16])
+		nd[0] = 'R' // echo server mark as replied
+		if err := op2p.GNetwork.WriteNode(0, nd); err != nil {
+			fmt.Println("write error:", err)
+			break
+		}
+	}
+}

go.mod

@@ -1,27 +1,38 @@
 module openp2p
 
-go 1.18
+go 1.20
 
 require (
-	github.com/gorilla/websocket v1.4.2
+	github.com/emirpasic/gods v1.18.1
-	github.com/kardianos/service v1.2.0
+	github.com/gorilla/websocket v1.5.3
-	github.com/lucas-clemente/quic-go v0.27.0
+	github.com/huin/goupnp v1.3.0
+	github.com/jackpal/go-nat-pmp v1.0.2
 	github.com/openp2p-cn/go-reuseport v0.3.2
-	golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f
+	github.com/openp2p-cn/service v1.0.0
+	github.com/openp2p-cn/totp v0.0.0-20230421034602-0f3320ffb25e
+	github.com/openp2p-cn/wireguard-go v0.0.20241020
+	github.com/quic-go/quic-go v0.34.0
+	github.com/vishvananda/netlink v1.1.1-0.20211118161826-650dca95af54
+	golang.org/x/net v0.30.0
+	golang.org/x/sys v0.26.0
+	golang.zx2c4.com/wireguard/windows v0.5.3
 )
 
 require (
-	github.com/cheekybits/genny v1.0.0 // indirect
-	github.com/fsnotify/fsnotify v1.4.9 // indirect
 	github.com/go-task/slim-sprig v0.0.0-20210107165309-348f09dbbbc0 // indirect
-	github.com/marten-seemann/qtls-go1-16 v0.1.5 // indirect
+	github.com/golang/mock v1.7.0-rc.1 // indirect
-	github.com/marten-seemann/qtls-go1-17 v0.1.1 // indirect
+	github.com/google/pprof v0.0.0-20210407192527-94a9f03dee38 // indirect
-	github.com/marten-seemann/qtls-go1-18 v0.1.1 // indirect
+	github.com/kardianos/service v1.2.2 // indirect
-	github.com/nxadm/tail v1.4.8 // indirect
+	github.com/onsi/ginkgo/v2 v2.2.0 // indirect
-	github.com/onsi/ginkgo v1.16.4 // indirect
+	github.com/quic-go/qtls-go1-19 v0.3.2 // indirect
-	golang.org/x/crypto v0.0.0-20210921155107-089bfa567519 // indirect
+	github.com/quic-go/qtls-go1-20 v0.2.2 // indirect
-	golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4 // indirect
+	github.com/vishvananda/netns v0.0.0-20210104183010-2eb08e3e575f // indirect
-	golang.org/x/net v0.0.0-20220722155237-a158d28d115b // indirect
+	golang.org/x/crypto v0.28.0 // indirect
-	golang.org/x/tools v0.1.12 // indirect
+	golang.org/x/exp v0.0.0-20230725093048-515e97ebf090 // indirect
-	gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7 // indirect
+	golang.org/x/mod v0.21.0 // indirect
+	golang.org/x/sync v0.8.0 // indirect
+	golang.org/x/tools v0.26.0 // indirect
+	golang.zx2c4.com/wintun v0.0.0-20230126152724-0fa3db229ce2 // indirect
+	golang.zx2c4.com/wireguard v0.0.0-20231211153847-12269c276173 // indirect
+	gvisor.dev/gvisor v0.0.0-20241128011400-745828301c93 // indirect
 )

go.sum

@@ -0,0 +1,109 @@
+github.com/chzyer/logex v1.1.10/go.mod h1:+Ywpsq7O8HXn0nuIou7OrIPyXbp3wmkHB+jjWRnGsAI=
+github.com/chzyer/readline v0.0.0-20180603132655-2972be24d48e/go.mod h1:nSuG5e5PlCu98SY8svDHJxuZscDgtXS6KTTbou5AhLI=
+github.com/chzyer/test v0.0.0-20180213035817-a1ea475d72b1/go.mod h1:Q3SI9o4m/ZMnBNeIyt5eFwwo7qiLfzFZmjNmxjkiQlU=
+github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
+github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
+github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
+github.com/emirpasic/gods v1.18.1 h1:FXtiHYKDGKCW2KzwZKx0iC0PQmdlorYgdFG9jPXJ1Bc=
+github.com/emirpasic/gods v1.18.1/go.mod h1:8tpGGwCnJ5H4r6BWwaV6OrWmMoPhUl5jm/FMNAnJvWQ=
+github.com/go-task/slim-sprig v0.0.0-20210107165309-348f09dbbbc0 h1:p104kn46Q8WdvHunIJ9dAyjPVtrBPhSr3KT2yUst43I=
+github.com/go-task/slim-sprig v0.0.0-20210107165309-348f09dbbbc0/go.mod h1:fyg7847qk6SyHyPtNmDHnmrv/HOrqktSC+C9fM+CJOE=
+github.com/golang/mock v1.7.0-rc.1 h1:YojYx61/OLFsiv6Rw1Z96LpldJIy31o+UHmwAUMJ6/U=
+github.com/golang/mock v1.7.0-rc.1/go.mod h1:s42URUywIqd+OcERslBJvOjepvNymP31m3q8d/GkuRs=
+github.com/golang/protobuf v1.5.4 h1:i7eJL8qZTpSEXOPTxNKhASYpMn+8e5Q6AdndVa1dWek=
+github.com/google/btree v1.1.2 h1:xf4v41cLI2Z6FxbKm+8Bu+m8ifhj15JuZ9sa0jZCMUU=
+github.com/google/go-cmp v0.6.0 h1:ofyhxvXcZhMsU5ulbFiLKl/XBFqE1GSq7atu8tAmTRI=
+github.com/google/pprof v0.0.0-20210407192527-94a9f03dee38 h1:yAJXTCF9TqKcTiHJAE8dj7HMvPfh66eeA2JYW7eFpSE=
+github.com/google/pprof v0.0.0-20210407192527-94a9f03dee38/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE=
+github.com/gorilla/websocket v1.5.3 h1:saDtZ6Pbx/0u+bgYQ3q96pZgCzfhKXGPqt7kZ72aNNg=
+github.com/gorilla/websocket v1.5.3/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE=
+github.com/huin/goupnp v1.3.0 h1:UvLUlWDNpoUdYzb2TCn+MuTWtcjXKSza2n6CBdQ0xXc=
+github.com/huin/goupnp v1.3.0/go.mod h1:gnGPsThkYa7bFi/KWmEysQRf48l2dvR5bxr2OFckNX8=
+github.com/ianlancetaylor/demangle v0.0.0-20200824232613-28f6c0f3b639/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc=
+github.com/jackpal/go-nat-pmp v1.0.2 h1:KzKSgb7qkJvOUTqYl9/Hg/me3pWgBmERKrTGD7BdWus=
+github.com/jackpal/go-nat-pmp v1.0.2/go.mod h1:QPH045xvCAeXUZOxsnwmrtiCoxIr9eob+4orBN1SBKc=
+github.com/kardianos/service v1.2.2 h1:ZvePhAHfvo0A7Mftk/tEzqEZ7Q4lgnR8sGz4xu1YX60=
+github.com/kardianos/service v1.2.2/go.mod h1:CIMRFEJVL+0DS1a3Nx06NaMn4Dz63Ng6O7dl0qH0zVM=
+github.com/onsi/ginkgo/v2 v2.2.0 h1:3ZNA3L1c5FYDFTTxbFeVGGD8jYvjYauHD30YgLxVsNI=
+github.com/onsi/ginkgo/v2 v2.2.0/go.mod h1:MEH45j8TBi6u9BMogfbp0stKC5cdGjumZj5Y7AG4VIk=
+github.com/onsi/gomega v1.20.1 h1:PA/3qinGoukvymdIDV8pii6tiZgC8kbmJO6Z5+b002Q=
+github.com/openp2p-cn/go-reuseport v0.3.2 h1:TO78WsyJ1F6g7rLp3hpTKOBxtZTU5Lz+Y4Mj+fVUfZc=
+github.com/openp2p-cn/go-reuseport v0.3.2/go.mod h1:+EwCusXz50jaYkPNZcCrK4cLoA9tr2jEiJC+bjzpWc8=
+github.com/openp2p-cn/service v1.0.0 h1:1++FroLvW4Mc/PStFIAF0mzudVW6E8EAeqWyIESTGZA=
+github.com/openp2p-cn/service v1.0.0/go.mod h1:U4VHekhSJldZ332W6bLviB1fipDrS4omY4dHVc/kgts=
+github.com/openp2p-cn/totp v0.0.0-20230421034602-0f3320ffb25e h1:QqP3Va/nPj45wq0C8OmGiyZ4HhbTcV6yGuhcYCMgbjg=
+github.com/openp2p-cn/totp v0.0.0-20230421034602-0f3320ffb25e/go.mod h1:RYVP3CTIvHD9IwQe2M3zy5iLKNjusRVDz/4gQuKcc/o=
+github.com/openp2p-cn/wireguard-go v0.0.20241020 h1:cNgG8o2ctYT9YanqalfMQo+jVju7MrdJFI6WLZZRr7M=
+github.com/openp2p-cn/wireguard-go v0.0.20241020/go.mod h1:ka26SCScyLEd+uFrnq6w4n65Sxq1W/xIJfXEXLLvJEc=
+github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
+github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
+github.com/quic-go/qtls-go1-19 v0.3.2 h1:tFxjCFcTQzK+oMxG6Zcvp4Dq8dx4yD3dDiIiyc86Z5U=
+github.com/quic-go/qtls-go1-19 v0.3.2/go.mod h1:ySOI96ew8lnoKPtSqx2BlI5wCpUVPT05RMAlajtnyOI=
+github.com/quic-go/qtls-go1-20 v0.2.2 h1:WLOPx6OY/hxtTxKV1Zrq20FtXtDEkeY00CGQm8GEa3E=
+github.com/quic-go/qtls-go1-20 v0.2.2/go.mod h1:JKtK6mjbAVcUTN/9jZpvLbGxvdWIKS8uT7EiStoU1SM=
+github.com/quic-go/quic-go v0.34.0 h1:OvOJ9LFjTySgwOTYUZmNoq0FzVicP8YujpV0kB7m2lU=
+github.com/quic-go/quic-go v0.34.0/go.mod h1:+4CVgVppm0FNjpG3UcX8Joi/frKOH7/ciD5yGcwOO1g=
+github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
+github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA=
+github.com/stretchr/testify v1.7.0 h1:nwc3DEeHmmLAfoZucVR881uASk0Mfjw8xYJ99tb5CcY=
+github.com/vishvananda/netlink v1.1.1-0.20211118161826-650dca95af54 h1:8mhqcHPqTMhSPoslhGYihEgSfc77+7La1P6kiB6+9So=
+github.com/vishvananda/netlink v1.1.1-0.20211118161826-650dca95af54/go.mod h1:twkDnbuQxJYemMlGd4JFIcuhgX83tXhKS2B/PRMpOho=
+github.com/vishvananda/netns v0.0.0-20200728191858-db3c7e526aae/go.mod h1:DD4vA1DwXk04H54A1oHXtwZmA0grkVMdPxx/VGLCah0=
+github.com/vishvananda/netns v0.0.0-20210104183010-2eb08e3e575f h1:p4VB7kIXpOQvVn1ZaTIVp+3vuYAXFe3OJEvjbUYJLaA=
+github.com/vishvananda/netns v0.0.0-20210104183010-2eb08e3e575f/go.mod h1:DD4vA1DwXk04H54A1oHXtwZmA0grkVMdPxx/VGLCah0=
+github.com/yuin/goldmark v1.4.1/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k=
+golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
+golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
+golang.org/x/crypto v0.28.0 h1:GBDwsMXVQi34v5CCYUm2jkJvu4cbtru2U4TN2PSyQnw=
+golang.org/x/crypto v0.28.0/go.mod h1:rmgy+3RHxRZMyY0jjAJShp2zgEdOqj2AO7U0pYmeQ7U=
+golang.org/x/exp v0.0.0-20230725093048-515e97ebf090 h1:Di6/M8l0O2lCLc6VVRWhgCiApHV8MnQurBnFSHsQtNY=
+golang.org/x/exp v0.0.0-20230725093048-515e97ebf090/go.mod h1:FXUEEKJgO7OQYeo8N01OfiKP8RXMtf6e8aTskBGqWdc=
+golang.org/x/mod v0.5.1/go.mod h1:5OXOZSfqPIIbmVBIIKWRFfZjPR0E5r58TLhUjH0a2Ro=
+golang.org/x/mod v0.21.0 h1:vvrHzRwRfVKSiLrG+d4FMl/Qi4ukBCE6kZlTUkDYRT0=
+golang.org/x/mod v0.21.0/go.mod h1:6SkKJ3Xj0I0BrPOZoBy3bdMptDDU9oJrpohJ3eWZ1fY=
+golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
+golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
+golang.org/x/net v0.0.0-20211015210444-4f30a5c0130f/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
+golang.org/x/net v0.30.0 h1:AcW1SDZMkb8IpzCdQUaIq2sP4sZ4zw+55h6ynffypl4=
+golang.org/x/net v0.30.0/go.mod h1:2wGyMJ5iFasEhkwi13ChkO/t1ECNC4X4eBKkVFyYFlU=
+golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
+golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
+golang.org/x/sync v0.8.0 h1:3NFvSEYkUoMifnESzZl15y791HH1qU2xm6eCJU5ZPXQ=
+golang.org/x/sync v0.8.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk=
+golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
+golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20191204072324-ce4227a45e2e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20200217220822-9197077df867/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20200728102440-3e129f6d46b1/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20201015000850-e3ed0017c211/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20211019181941-9d821ace8654/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.26.0 h1:KHjCJyddX0LoSTb3J+vWpupP9p0oznkqVk/IfjymZbo=
+golang.org/x/sys v0.26.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
+golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
+golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
+golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
+golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ=
+golang.org/x/text v0.19.0 h1:kTxAhCbGbxhK0IwgSKiMO5awPoDQ0RpfiVYBfK860YM=
+golang.org/x/time v0.7.0 h1:ntUhktv3OPE6TgYxXWv9vKvUSJyIFJlyohwbkEwPrKQ=
+golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
+golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
+golang.org/x/tools v0.1.8/go.mod h1:nABZi5QlRsZVlzPpHl034qft6wpY4eDcsTt5AaioBiU=
+golang.org/x/tools v0.26.0 h1:v/60pFQmzmT9ExmjDv2gGIfi3OqfKoEP6I5+umXlbnQ=
+golang.org/x/tools v0.26.0/go.mod h1:TPVVj70c7JJ3WCazhD8OdXcZg/og+b9+tH/KxylGwH0=
+golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
+golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
+golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
+golang.zx2c4.com/wintun v0.0.0-20230126152724-0fa3db229ce2 h1:B82qJJgjvYKsXS9jeunTOisW56dUokqW/FOteYJJ/yg=
+golang.zx2c4.com/wintun v0.0.0-20230126152724-0fa3db229ce2/go.mod h1:deeaetjYA+DHMHg+sMSMI58GrEteJUUzzw7en6TJQcI=
+golang.zx2c4.com/wireguard v0.0.0-20231211153847-12269c276173 h1:/jFs0duh4rdb8uIfPMv78iAJGcPKDeqAFnaLBropIC4=
+golang.zx2c4.com/wireguard v0.0.0-20231211153847-12269c276173/go.mod h1:tkCQ4FQXmpAgYVh++1cq16/dH4QJtmvpRv19DWGAHSA=
+golang.zx2c4.com/wireguard/windows v0.5.3 h1:On6j2Rpn3OEMXqBq00QEDC7bWSZrPIHKIus8eIuExIE=
+golang.zx2c4.com/wireguard/windows v0.5.3/go.mod h1:9TEe8TJmtwyQebdFwAkEWOPr3prrtqm+REGFifP60hI=
+google.golang.org/protobuf v1.33.0 h1:uNO2rsAINq/JlFpSdYEKIZ0uKD/R9cpdv0T+yoGwGmI=
+gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
+gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
+gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=
+gvisor.dev/gvisor v0.0.0-20241128011400-745828301c93 h1:QyA/pFgC67EZ5+0oRfiNFhfEGd3NqZM1A2HQEuPKC3c=
+gvisor.dev/gvisor v0.0.0-20241128011400-745828301c93/go.mod h1:5DMfjtclAbTIjbXqO1qCe2K5GKKxWz2JHvCChuTcJEM=

lib/openp2p.go

@@ -0,0 +1,18 @@
+package main
+
+// On Windows env
+// cd lib
+// go build -o openp2p.dll -buildmode=c-shared openp2p.go
+// caller example see example/dll
+import (
+	op "openp2p/core"
+)
+import "C"
+
+func main() {
+}
+
+//export RunCmd
+func RunCmd(cmd *C.char) {
+	op.RunCmd(C.GoString(cmd))
+}

pkg/upnp/nat.go

@@ -0,0 +1,239 @@
+// Copyright 2015 The go-ethereum Authors
+// This file is part of the go-ethereum library.
+//
+// The go-ethereum library is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Lesser General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// The go-ethereum library is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU Lesser General Public License for more details.
+//
+// You should have received a copy of the GNU Lesser General Public License
+// along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>.
+
+// Package nat provides access to common network port mapping protocols.
+package openp2p
+
+import (
+	"errors"
+	"fmt"
+	"net"
+	"strings"
+	"sync"
+	"time"
+
+	natpmp "github.com/jackpal/go-nat-pmp"
+)
+
+// Interface An implementation of nat.Interface can map local ports to ports
+// accessible from the Internet.
+type Interface interface {
+	// These methods manage a mapping between a port on the local
+	// machine to a port that can be connected to from the internet.
+	//
+	// protocol is "UDP" or "TCP". Some implementations allow setting
+	// a display name for the mapping. The mapping may be removed by
+	// the gateway when its lifetime ends.
+	AddMapping(protocol string, extport, intport int, name string, lifetime time.Duration) (uint16, error)
+	DeleteMapping(protocol string, extport, intport int) error
+
+	// ExternalIP should return the external (Internet-facing)
+	// address of the gateway device.
+	ExternalIP() (net.IP, error)
+
+	// String should return name of the method. This is used for logging.
+	String() string
+}
+
+// Parse parses a NAT interface description.
+// The following formats are currently accepted.
+// Note that mechanism names are not case-sensitive.
+//
+//	"" or "none"         return nil
+//	"extip:77.12.33.4"   will assume the local machine is reachable on the given IP
+//	"any"                uses the first auto-detected mechanism
+//	"upnp"               uses the Universal Plug and Play protocol
+//	"pmp"                uses NAT-PMP with an auto-detected gateway address
+//	"pmp:192.168.0.1"    uses NAT-PMP with the given gateway address
+func Parse(spec string) (Interface, error) {
+	var (
+		before, after, found = strings.Cut(spec, ":")
+		mech                 = strings.ToLower(before)
+		ip                   net.IP
+	)
+	if found {
+		ip = net.ParseIP(after)
+		if ip == nil {
+			return nil, errors.New("invalid IP address")
+		}
+	}
+	switch mech {
+	case "", "none", "off":
+		return nil, nil
+	case "any", "auto", "on":
+		return Any(), nil
+	case "extip", "ip":
+		if ip == nil {
+			return nil, errors.New("missing IP address")
+		}
+		return ExtIP(ip), nil
+	case "upnp":
+		return UPnP(), nil
+	case "pmp", "natpmp", "nat-pmp":
+		return PMP(ip), nil
+	default:
+		return nil, fmt.Errorf("unknown mechanism %q", before)
+	}
+}
+
+const (
+	DefaultMapTimeout = 10 * time.Minute
+)
+
+// Map adds a port mapping on m and keeps it alive until c is closed.
+// This function is typically invoked in its own goroutine.
+//
+// Note that Map does not handle the situation where the NAT interface assigns a different
+// external port than the requested one.
+func Map(m Interface, c <-chan struct{}, protocol string, extport, intport int, name string) {
+	// log := log.New("proto", protocol, "extport", extport, "intport", intport, "interface", m)
+	refresh := time.NewTimer(DefaultMapTimeout)
+	defer func() {
+		refresh.Stop()
+		// log.Debug("Deleting port mapping")
+		m.DeleteMapping(protocol, extport, intport)
+	}()
+	if _, err := m.AddMapping(protocol, extport, intport, name, DefaultMapTimeout); err != nil {
+		// log.Debug("Couldn't add port mapping", "err", err)
+	} else {
+		// log.Info("Mapped network port")
+	}
+	for {
+		select {
+		case _, ok := <-c:
+			if !ok {
+				return
+			}
+		case <-refresh.C:
+			// log.Trace("Refreshing port mapping")
+			if _, err := m.AddMapping(protocol, extport, intport, name, DefaultMapTimeout); err != nil {
+				// log.Debug("Couldn't add port mapping", "err", err)
+			}
+			refresh.Reset(DefaultMapTimeout)
+		}
+	}
+}
+
+// ExtIP assumes that the local machine is reachable on the given
+// external IP address, and that any required ports were mapped manually.
+// Mapping operations will not return an error but won't actually do anything.
+type ExtIP net.IP
+
+func (n ExtIP) ExternalIP() (net.IP, error) { return net.IP(n), nil }
+func (n ExtIP) String() string              { return fmt.Sprintf("ExtIP(%v)", net.IP(n)) }
+
+// These do nothing.
+
+func (ExtIP) AddMapping(string, int, int, string, time.Duration) (uint16, error) { return 0, nil }
+func (ExtIP) DeleteMapping(string, int, int) error                               { return nil }
+
+// Any returns a port mapper that tries to discover any supported
+// mechanism on the local network.
+func Any() Interface {
+	// TODO: attempt to discover whether the local machine has an
+	// Internet-class address. Return ExtIP in this case.
+	return startautodisc("UPnP or NAT-PMP", func() Interface {
+		found := make(chan Interface, 2)
+		go func() { found <- discoverUPnP() }()
+		go func() { found <- discoverPMP() }()
+		for i := 0; i < cap(found); i++ {
+			if c := <-found; c != nil {
+				return c
+			}
+		}
+		return nil
+	})
+}
+
+// UPnP returns a port mapper that uses UPnP. It will attempt to
+// discover the address of your router using UDP broadcasts.
+func UPnP() Interface {
+	return startautodisc("UPnP", discoverUPnP)
+}
+
+// PMP returns a port mapper that uses NAT-PMP. The provided gateway
+// address should be the IP of your router. If the given gateway
+// address is nil, PMP will attempt to auto-discover the router.
+func PMP(gateway net.IP) Interface {
+	if gateway != nil {
+		return &pmp{gw: gateway, c: natpmp.NewClient(gateway)}
+	}
+	return startautodisc("NAT-PMP", discoverPMP)
+}
+
+// autodisc represents a port mapping mechanism that is still being
+// auto-discovered. Calls to the Interface methods on this type will
+// wait until the discovery is done and then call the method on the
+// discovered mechanism.
+//
+// This type is useful because discovery can take a while but we
+// want return an Interface value from UPnP, PMP and Auto immediately.
+type autodisc struct {
+	what string // type of interface being autodiscovered
+	once sync.Once
+	doit func() Interface
+
+	mu    sync.Mutex
+	found Interface
+}
+
+func startautodisc(what string, doit func() Interface) Interface {
+	// TODO: monitor network configuration and rerun doit when it changes.
+	return &autodisc{what: what, doit: doit}
+}
+
+func (n *autodisc) AddMapping(protocol string, extport, intport int, name string, lifetime time.Duration) (uint16, error) {
+	if err := n.wait(); err != nil {
+		return 0, err
+	}
+	return n.found.AddMapping(protocol, extport, intport, name, lifetime)
+}
+
+func (n *autodisc) DeleteMapping(protocol string, extport, intport int) error {
+	if err := n.wait(); err != nil {
+		return err
+	}
+	return n.found.DeleteMapping(protocol, extport, intport)
+}
+
+func (n *autodisc) ExternalIP() (net.IP, error) {
+	if err := n.wait(); err != nil {
+		return nil, err
+	}
+	return n.found.ExternalIP()
+}
+
+func (n *autodisc) String() string {
+	n.mu.Lock()
+	defer n.mu.Unlock()
+	if n.found == nil {
+		return n.what
+	}
+	return n.found.String()
+}
+
+// wait blocks until auto-discovery has been performed.
+func (n *autodisc) wait() error {
+	n.once.Do(func() {
+		n.mu.Lock()
+		n.found = n.doit()
+		n.mu.Unlock()
+	})
+	if n.found == nil {
+		return fmt.Errorf("no %s router discovered", n.what)
+	}
+	return nil
+}

pkg/upnp/natpmp.go

@@ -0,0 +1,130 @@
+// Copyright 2015 The go-ethereum Authors
+// This file is part of the go-ethereum library.
+//
+// The go-ethereum library is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Lesser General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// The go-ethereum library is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU Lesser General Public License for more details.
+//
+// You should have received a copy of the GNU Lesser General Public License
+// along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>.
+
+package openp2p
+
+import (
+	"fmt"
+	"net"
+	"strings"
+	"time"
+
+	natpmp "github.com/jackpal/go-nat-pmp"
+)
+
+// natPMPClient adapts the NAT-PMP protocol implementation so it conforms to
+// the common interface.
+type pmp struct {
+	gw net.IP
+	c  *natpmp.Client
+}
+
+func (n *pmp) String() string {
+	return fmt.Sprintf("NAT-PMP(%v)", n.gw)
+}
+
+func (n *pmp) ExternalIP() (net.IP, error) {
+	response, err := n.c.GetExternalAddress()
+	if err != nil {
+		return nil, err
+	}
+	return response.ExternalIPAddress[:], nil
+}
+
+func (n *pmp) AddMapping(protocol string, extport, intport int, name string, lifetime time.Duration) (uint16, error) {
+	if lifetime <= 0 {
+		return 0, fmt.Errorf("lifetime must not be <= 0")
+	}
+	// Note order of port arguments is switched between our
+	// AddMapping and the client's AddPortMapping.
+	res, err := n.c.AddPortMapping(strings.ToLower(protocol), intport, extport, int(lifetime/time.Second))
+	if err != nil {
+		return 0, err
+	}
+
+	// NAT-PMP maps an alternative available port number if the requested port
+	// is already mapped to another address and returns success. Handling of
+	// alternate port numbers is done by the caller.
+	return res.MappedExternalPort, nil
+}
+
+func (n *pmp) DeleteMapping(protocol string, extport, intport int) (err error) {
+	// To destroy a mapping, send an add-port with an internalPort of
+	// the internal port to destroy, an external port of zero and a
+	// time of zero.
+	_, err = n.c.AddPortMapping(strings.ToLower(protocol), intport, 0, 0)
+	return err
+}
+
+func discoverPMP() Interface {
+	// run external address lookups on all potential gateways
+	gws := potentialGateways()
+	found := make(chan *pmp, len(gws))
+	for i := range gws {
+		gw := gws[i]
+		go func() {
+			c := natpmp.NewClient(gw)
+			if _, err := c.GetExternalAddress(); err != nil {
+				found <- nil
+			} else {
+				found <- &pmp{gw, c}
+			}
+		}()
+	}
+	// return the one that responds first.
+	// discovery needs to be quick, so we stop caring about
+	// any responses after a very short timeout.
+	timeout := time.NewTimer(1 * time.Second)
+	defer timeout.Stop()
+	for range gws {
+		select {
+		case c := <-found:
+			if c != nil {
+				return c
+			}
+		case <-timeout.C:
+			return nil
+		}
+	}
+	return nil
+}
+
+// TODO: improve this. We currently assume that (on most networks)
+// the router is X.X.X.1 in a local LAN range.
+func potentialGateways() (gws []net.IP) {
+	ifaces, err := net.Interfaces()
+	if err != nil {
+		return nil
+	}
+	for _, iface := range ifaces {
+		ifaddrs, err := iface.Addrs()
+		if err != nil {
+			return gws
+		}
+		for _, addr := range ifaddrs {
+			if x, ok := addr.(*net.IPNet); ok {
+				if x.IP.IsPrivate() {
+					ip := x.IP.Mask(x.Mask).To4()
+					if ip != nil {
+						ip[3] = ip[3] | 0x01
+						gws = append(gws, ip)
+					}
+				}
+			}
+		}
+	}
+	return gws
+}

pkg/upnp/natupnp.go

@@ -0,0 +1,247 @@
+// Copyright 2015 The go-ethereum Authors
+// This file is part of the go-ethereum library.
+//
+// The go-ethereum library is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Lesser General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// The go-ethereum library is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU Lesser General Public License for more details.
+//
+// You should have received a copy of the GNU Lesser General Public License
+// along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>.
+
+package openp2p
+
+import (
+	"errors"
+	"fmt"
+	"math"
+	"math/rand"
+	"net"
+	"strings"
+	"sync"
+	"time"
+
+	"github.com/huin/goupnp"
+	"github.com/huin/goupnp/dcps/internetgateway1"
+	"github.com/huin/goupnp/dcps/internetgateway2"
+)
+
+const (
+	soapRequestTimeout = 3 * time.Second
+	rateLimit          = 200 * time.Millisecond
+)
+
+type upnp struct {
+	dev         *goupnp.RootDevice
+	service     string
+	client      upnpClient
+	mu          sync.Mutex
+	lastReqTime time.Time
+	rand        *rand.Rand
+}
+
+type upnpClient interface {
+	GetExternalIPAddress() (string, error)
+	AddPortMapping(string, uint16, string, uint16, string, bool, string, uint32) error
+	DeletePortMapping(string, uint16, string) error
+	GetNATRSIPStatus() (sip bool, nat bool, err error)
+}
+
+func (n *upnp) natEnabled() bool {
+	var ok bool
+	var err error
+	n.withRateLimit(func() error {
+		_, ok, err = n.client.GetNATRSIPStatus()
+		return err
+	})
+	return err == nil && ok
+}
+
+func (n *upnp) ExternalIP() (addr net.IP, err error) {
+	var ipString string
+	n.withRateLimit(func() error {
+		ipString, err = n.client.GetExternalIPAddress()
+		return err
+	})
+
+	if err != nil {
+		return nil, err
+	}
+	ip := net.ParseIP(ipString)
+	if ip == nil {
+		return nil, errors.New("bad IP in response")
+	}
+	return ip, nil
+}
+
+func (n *upnp) AddMapping(protocol string, extport, intport int, desc string, lifetime time.Duration) (uint16, error) {
+	ip, err := n.internalAddress()
+	if err != nil {
+		return 0, nil // TODO: Shouldn't we return the error?
+	}
+	protocol = strings.ToUpper(protocol)
+	lifetimeS := uint32(lifetime / time.Second)
+	n.DeleteMapping(protocol, extport, intport)
+
+	err = n.withRateLimit(func() error {
+		return n.client.AddPortMapping("", uint16(extport), protocol, uint16(intport), ip.String(), true, desc, lifetimeS)
+	})
+	if err == nil {
+		return uint16(extport), nil
+	}
+
+	return uint16(extport), n.withRateLimit(func() error {
+		p, err := n.addAnyPortMapping(protocol, extport, intport, ip, desc, lifetimeS)
+		if err == nil {
+			extport = int(p)
+		}
+		return err
+	})
+}
+
+func (n *upnp) addAnyPortMapping(protocol string, extport, intport int, ip net.IP, desc string, lifetimeS uint32) (uint16, error) {
+	if client, ok := n.client.(*internetgateway2.WANIPConnection2); ok {
+		return client.AddAnyPortMapping("", uint16(extport), protocol, uint16(intport), ip.String(), true, desc, lifetimeS)
+	}
+	// It will retry with a random port number if the client does
+	// not support AddAnyPortMapping.
+	extport = n.randomPort()
+	err := n.client.AddPortMapping("", uint16(extport), protocol, uint16(intport), ip.String(), true, desc, lifetimeS)
+	if err != nil {
+		return 0, err
+	}
+	return uint16(extport), nil
+}
+
+func (n *upnp) randomPort() int {
+	if n.rand == nil {
+		n.rand = rand.New(rand.NewSource(time.Now().UnixNano()))
+	}
+	return n.rand.Intn(math.MaxUint16-10000) + 10000
+}
+
+func (n *upnp) internalAddress() (net.IP, error) {
+	devaddr, err := net.ResolveUDPAddr("udp4", n.dev.URLBase.Host)
+	if err != nil {
+		return nil, err
+	}
+	ifaces, err := net.Interfaces()
+	if err != nil {
+		return nil, err
+	}
+	for _, iface := range ifaces {
+		addrs, err := iface.Addrs()
+		if err != nil {
+			return nil, err
+		}
+		for _, addr := range addrs {
+			if x, ok := addr.(*net.IPNet); ok && x.Contains(devaddr.IP) {
+				return x.IP, nil
+			}
+		}
+	}
+	return nil, fmt.Errorf("could not find local address in same net as %v", devaddr)
+}
+
+func (n *upnp) DeleteMapping(protocol string, extport, intport int) error {
+	return n.withRateLimit(func() error {
+		return n.client.DeletePortMapping("", uint16(extport), strings.ToUpper(protocol))
+	})
+}
+
+func (n *upnp) String() string {
+	return "UPNP " + n.service
+}
+
+func (n *upnp) withRateLimit(fn func() error) error {
+	n.mu.Lock()
+	defer n.mu.Unlock()
+
+	lastreq := time.Since(n.lastReqTime)
+	if lastreq < rateLimit {
+		time.Sleep(rateLimit - lastreq)
+	}
+	err := fn()
+	n.lastReqTime = time.Now()
+	return err
+}
+
+// discoverUPnP searches for Internet Gateway Devices
+// and returns the first one it can find on the local network.
+func discoverUPnP() Interface {
+	found := make(chan *upnp, 2)
+	// IGDv1
+	go discover(found, internetgateway1.URN_WANConnectionDevice_1, func(sc goupnp.ServiceClient) *upnp {
+		switch sc.Service.ServiceType {
+		case internetgateway1.URN_WANIPConnection_1:
+			return &upnp{service: "IGDv1-IP1", client: &internetgateway1.WANIPConnection1{ServiceClient: sc}}
+		case internetgateway1.URN_WANPPPConnection_1:
+			return &upnp{service: "IGDv1-PPP1", client: &internetgateway1.WANPPPConnection1{ServiceClient: sc}}
+		}
+		return nil
+	})
+	// IGDv2
+	go discover(found, internetgateway2.URN_WANConnectionDevice_2, func(sc goupnp.ServiceClient) *upnp {
+		switch sc.Service.ServiceType {
+		case internetgateway2.URN_WANIPConnection_1:
+			return &upnp{service: "IGDv2-IP1", client: &internetgateway2.WANIPConnection1{ServiceClient: sc}}
+		case internetgateway2.URN_WANIPConnection_2:
+			return &upnp{service: "IGDv2-IP2", client: &internetgateway2.WANIPConnection2{ServiceClient: sc}}
+		case internetgateway2.URN_WANPPPConnection_1:
+			return &upnp{service: "IGDv2-PPP1", client: &internetgateway2.WANPPPConnection1{ServiceClient: sc}}
+		}
+		return nil
+	})
+	for i := 0; i < cap(found); i++ {
+		if c := <-found; c != nil {
+			return c
+		}
+	}
+	return nil
+}
+
+// finds devices matching the given target and calls matcher for all
+// advertised services of each device. The first non-nil service found
+// is sent into out. If no service matched, nil is sent.
+func discover(out chan<- *upnp, target string, matcher func(goupnp.ServiceClient) *upnp) {
+	devs, err := goupnp.DiscoverDevices(target)
+	if err != nil {
+		out <- nil
+		return
+	}
+	found := false
+	for i := 0; i < len(devs) && !found; i++ {
+		if devs[i].Root == nil {
+			continue
+		}
+		devs[i].Root.Device.VisitServices(func(service *goupnp.Service) {
+			if found {
+				return
+			}
+			// check for a matching IGD service
+			sc := goupnp.ServiceClient{
+				SOAPClient: service.NewSOAPClient(),
+				RootDevice: devs[i].Root,
+				Location:   devs[i].Location,
+				Service:    service,
+			}
+			sc.SOAPClient.HTTPClient.Timeout = soapRequestTimeout
+			upnp := matcher(sc)
+			if upnp == nil {
+				return
+			}
+			upnp.dev = devs[i].Root
+
+			out <- upnp
+			found = true
+		})
+	}
+	if !found {
+		out <- nil
+	}
+}