landaiqing/voidraft
1
1
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases 4 Wiki Activity

✨ Use guesslang to detect the language

Browse Source
This commit is contained in:
landaiqing
2025-06-19 16:55:02 +08:00
parent f2894b2a89
commit 25858cb42b
12 changed files with 724 additions and 790 deletions
Download Patch File Download Diff File Expand all files Collapse all files

87
frontend/src/components/toolbar/BlockLanguageSelector.vue
View File

@@ -13,28 +13,71 @@ const searchInputRef = ref<HTMLInputElement>();
// 语言别名映射
const LANGUAGE_ALIASES: Record<SupportedLanguage, string> = {
text: 'txt',
javascript: 'js',
typescript: 'ts',
python: 'py',
html: 'htm',
css: '',
json: '',
markdown: 'md',
shell: 'sh',
sql: '',
yaml: 'yml',
xml: '',
php: '',
java: '',
json: 'JSON',
py: 'python',
html: 'HTML',
sql: 'SQL',
md: 'markdown',
java: 'Java',
php: 'PHP',
css: 'CSS',
xml: 'XML',
cpp: 'c++',
c: '',
go: '',
rust: 'rs',
ruby: 'rb'
rs: 'rust',
cs: 'c#',
rb: 'ruby',
sh: 'shell',
yaml: 'yml',
toml: 'TOML',
go: 'Go',
clj: 'clojure',
ex: 'elixir',
erl: 'erlang',
js: 'javascript',
ts: 'typescript',
swift: 'Swift',
kt: 'kotlin',
groovy: 'Groovy',
ps1: 'powershell',
dart: 'Dart',
scala: 'Scala'
};
// 语言显示名称映射
const LANGUAGE_NAMES: Record<SupportedLanguage, string> = {
text: 'Plain Text',
json: 'JSON',
py: 'Python',
html: 'HTML',
sql: 'SQL',
md: 'Markdown',
java: 'Java',
php: 'PHP',
css: 'CSS',
xml: 'XML',
cpp: 'C++',
rs: 'Rust',
cs: 'C#',
rb: 'Ruby',
sh: 'Shell',
yaml: 'YAML',
toml: 'TOML',
go: 'Go',
clj: 'Clojure',
ex: 'Elixir',
erl: 'Erlang',
js: 'JavaScript',
ts: 'TypeScript',
swift: 'Swift',
kt: 'Kotlin',
groovy: 'Groovy',
ps1: 'PowerShell',
dart: 'Dart',
scala: 'Scala'
};
// 当前选中的语言
const currentLanguage = ref<SupportedLanguage>('javascript');
const currentLanguage = ref<SupportedLanguage>('text');
// 过滤后的语言列表
const filteredLanguages = computed(() => {
@@ -44,8 +87,10 @@ const filteredLanguages = computed(() => {
const query = searchQuery.value.toLowerCase();
return SUPPORTED_LANGUAGES.filter(langId => {
const name = LANGUAGE_NAMES[langId];
const alias = LANGUAGE_ALIASES[langId];
return langId.toLowerCase().includes(query) ||
(name && name.toLowerCase().includes(query)) ||
(alias && alias.toLowerCase().includes(query));
});
});
@@ -96,7 +141,7 @@ onUnmounted(() => {
// 获取当前语言的显示名称
const getCurrentLanguageName = computed(() => {
return currentLanguage.value;
return LANGUAGE_NAMES[currentLanguage.value] || currentLanguage.value;
});
</script>
@@ -143,8 +188,8 @@ const getCurrentLanguageName = computed(() => {
:class="{ 'active': currentLanguage === language }"
@click="selectLanguage(language)"
>
<span class="language-name">{{ language }}</span>
<span class="language-alias" v-if="LANGUAGE_ALIASES[language]">{{ LANGUAGE_ALIASES[language] }}</span>
<span class="language-name">{{ LANGUAGE_NAMES[language] || language }}</span>
<span class="language-alias">{{ language }}</span>
</div>
<!-- 无结果提示 -->

27
frontend/src/views/editor/extensions/codeblock/index.ts
View File

@@ -1,33 +1,32 @@
/**
* CodeBlock 扩展主入口
*
*
* 配置说明：
* - showBackground: 控制是否显示代码块的背景色区分
* - enableAutoDetection: 控制是否启用内容的语言自动检测功能
* - defaultLanguage: 新建代码块时使用的默认语言（也是自动检测的回退语言）
* - defaultAutoDetect: 新建代码块时是否默认添加-a标记启用自动检测
*
*
* 注意：defaultLanguage 和 defaultAutoDetect 是配合使用的：
* - 如果 defaultAutoDetect=true，新建块会是 ∞∞∞javascript-a（会根据内容自动检测语言）
* - 如果 defaultAutoDetect=false，新建块会是 ∞∞∞javascript（固定使用指定语言）
*/
import {Extension} from '@codemirror/state';
import {keymap} from '@codemirror/view';
import {keymap, lineNumbers} from '@codemirror/view';
// 导入核心模块
import {blockState} from './state';
import {getBlockDecorationExtensions} from './decorations';
import * as commands from './commands';
import {selectAll, getBlockSelectExtensions} from './selectAll';
import {getBlockSelectExtensions, selectAll} from './selectAll';
import {getCopyPasteExtensions, getCopyPasteKeymap} from './copyPaste';
import {deleteLineCommand} from './deleteLine';
import {moveLineUp, moveLineDown} from './moveLines';
import {moveLineDown, moveLineUp} from './moveLines';
import {transposeChars} from './transposeChars';
import {getCodeBlockLanguageExtension} from './lang-parser';
import {createLanguageDetection} from './language-detection';
import {EditorOptions, SupportedLanguage} from './types';
import {lineNumbers} from '@codemirror/view';
/**
* 代码块扩展配置选项
@@ -38,10 +37,10 @@ export interface CodeBlockOptions {
/** 是否启用语言自动检测功能 */
enableAutoDetection?: boolean;
/** 新建块时的默认语言 */
defaultLanguage?: SupportedLanguage;
/** 新建块时是否默认启用自动检测（添加-a标记） */
defaultAutoDetect?: boolean;
}
@@ -52,10 +51,10 @@ export interface CodeBlockOptions {
function getBlockLineFromPos(state: any, pos: number) {
const line = state.doc.lineAt(pos);
const blocks = state.field(blockState);
const block = blocks.find((block: any) =>
const block = blocks.find((block: any) =>
block.content.from <= line.from && block.content.to >= line.from
);
if (block) {
const firstBlockLine = state.doc.lineAt(block.content.from).number;
return {
@@ -112,8 +111,8 @@ export function createCodeBlockExtension(options: CodeBlockOptions = {}): Extens
// 语言自动检测（如果启用）
...(enableAutoDetection ? [createLanguageDetection({
defaultLanguage: defaultLanguage,
confidenceThreshold: 0.3,
minContentLength: 8,
confidenceThreshold: 0.15,
minContentLength: 8
})] : []),
// 视觉装饰系统
@@ -297,14 +296,12 @@ export {
createLanguageDetection,
detectLanguage,
detectLanguages,
detectLanguageHeuristic,
getSupportedDetectionLanguages,
levenshteinDistance,
type LanguageDetectionResult
} from './language-detection';
// 行号相关
export { getBlockLineFromPos, blockLineNumbers };
export {getBlockLineFromPos, blockLineNumbers};
/**
* 默认导出

64
frontend/src/views/editor/extensions/codeblock/lang-parser/languages.ts
View File

@@ -14,12 +14,21 @@ import { cssLanguage } from "@codemirror/lang-css";
import { cppLanguage } from "@codemirror/lang-cpp";
import { xmlLanguage } from "@codemirror/lang-xml";
import { rustLanguage } from "@codemirror/lang-rust";
import { yamlLanguage } from "@codemirror/lang-yaml";
import { StreamLanguage } from "@codemirror/language";
import { ruby } from "@codemirror/legacy-modes/mode/ruby";
import { shell } from "@codemirror/legacy-modes/mode/shell";
import { go } from "@codemirror/legacy-modes/mode/go";
import { yamlLanguage } from "@codemirror/lang-yaml";
import { csharp } from "@codemirror/legacy-modes/mode/clike";
import { clojure } from "@codemirror/legacy-modes/mode/clojure";
import { erlang } from "@codemirror/legacy-modes/mode/erlang";
import { swift } from "@codemirror/legacy-modes/mode/swift";
import { kotlin } from "@codemirror/legacy-modes/mode/clike";
import { groovy } from "@codemirror/legacy-modes/mode/groovy";
import { powerShell } from "@codemirror/legacy-modes/mode/powershell";
import { scala } from "@codemirror/legacy-modes/mode/clike";
import { toml } from "@codemirror/legacy-modes/mode/toml";
import { SupportedLanguage } from '../types';
@@ -30,34 +39,43 @@ export class LanguageInfo {
constructor(
public token: SupportedLanguage,
public name: string,
public parser: any,
public guesslang?: string | null
public parser: any
) {}
}
/**
* 支持的语言列表
* 支持的语言列表（与 Worker 中的 LANGUAGES 对应）
*/
export const LANGUAGES: LanguageInfo[] = [
new LanguageInfo("text", "Plain Text", null),
new LanguageInfo("json", "JSON", jsonLanguage.parser, "json"),
new LanguageInfo("python", "Python", pythonLanguage.parser, "py"),
new LanguageInfo("javascript", "JavaScript", javascriptLanguage.parser, "js"),
new LanguageInfo("typescript", "TypeScript", typescriptLanguage.parser, "ts"),
new LanguageInfo("html", "HTML", htmlLanguage.parser, "html"),
new LanguageInfo("css", "CSS", cssLanguage.parser, "css"),
new LanguageInfo("sql", "SQL", StandardSQL.language.parser, "sql"),
new LanguageInfo("markdown", "Markdown", markdownLanguage.parser, "md"),
new LanguageInfo("java", "Java", javaLanguage.parser, "java"),
new LanguageInfo("php", "PHP", phpLanguage.configure({top:"Program"}).parser, "php"),
new LanguageInfo("xml", "XML", xmlLanguage.parser, "xml"),
new LanguageInfo("cpp", "C++", cppLanguage.parser, "cpp"),
new LanguageInfo("c", "C", cppLanguage.parser, "c"),
new LanguageInfo("rust", "Rust", rustLanguage.parser, "rs"),
new LanguageInfo("ruby", "Ruby", StreamLanguage.define(ruby).parser, "rb"),
new LanguageInfo("shell", "Shell", StreamLanguage.define(shell).parser, "sh"),
new LanguageInfo("yaml", "YAML", yamlLanguage.parser, "yaml"),
new LanguageInfo("go", "Go", StreamLanguage.define(go).parser, "go"),
new LanguageInfo("json", "JSON", jsonLanguage.parser),
new LanguageInfo("py", "Python", pythonLanguage.parser),
new LanguageInfo("html", "HTML", htmlLanguage.parser),
new LanguageInfo("sql", "SQL", StandardSQL.language.parser),
new LanguageInfo("md", "Markdown", markdownLanguage.parser),
new LanguageInfo("java", "Java", javaLanguage.parser),
new LanguageInfo("php", "PHP", phpLanguage.configure({top:"Program"}).parser),
new LanguageInfo("css", "CSS", cssLanguage.parser),
new LanguageInfo("xml", "XML", xmlLanguage.parser),
new LanguageInfo("cpp", "C++", cppLanguage.parser),
new LanguageInfo("rs", "Rust", rustLanguage.parser),
new LanguageInfo("cs", "C#", StreamLanguage.define(csharp).parser),
new LanguageInfo("rb", "Ruby", StreamLanguage.define(ruby).parser),
new LanguageInfo("sh", "Shell", StreamLanguage.define(shell).parser),
new LanguageInfo("yaml", "YAML", yamlLanguage.parser),
new LanguageInfo("toml", "TOML", StreamLanguage.define(toml).parser),
new LanguageInfo("go", "Go", StreamLanguage.define(go).parser),
new LanguageInfo("clj", "Clojure", StreamLanguage.define(clojure).parser),
new LanguageInfo("ex", "Elixir", null), // 暂无解析器
new LanguageInfo("erl", "Erlang", StreamLanguage.define(erlang).parser),
new LanguageInfo("js", "JavaScript", javascriptLanguage.parser),
new LanguageInfo("ts", "TypeScript", typescriptLanguage.parser),
new LanguageInfo("swift", "Swift", StreamLanguage.define(swift).parser),
new LanguageInfo("kt", "Kotlin", StreamLanguage.define(kotlin).parser),
new LanguageInfo("groovy", "Groovy", StreamLanguage.define(groovy).parser),
new LanguageInfo("ps1", "PowerShell", StreamLanguage.define(powerShell).parser),
new LanguageInfo("dart", "Dart", null), // 暂无解析器
new LanguageInfo("scala", "Scala", StreamLanguage.define(scala).parser),
];
/**
@@ -79,4 +97,4 @@ export function getLanguage(token: SupportedLanguage): LanguageInfo | undefined
*/
export function getLanguageTokens(): SupportedLanguage[] {
return LANGUAGES.map(lang => lang.token);
}
}

366
frontend/src/views/editor/extensions/codeblock/language-detection/autodetect.ts
View File

@@ -1,62 +1,92 @@
/**
* 自动语言检测
* 基于内容变化自动检测和更新代码块语言
* 基于 Web Worker 的语言自动检测
*/
import { EditorState, Annotation } from '@codemirror/state';
import { EditorView, ViewPlugin } from '@codemirror/view';
import { blockState, getActiveNoteBlock } from '../state';
import { blockState } from '../state';
import { levenshteinDistance } from './levenshtein';
import { detectLanguageHeuristic, LanguageDetectionResult } from './heuristics';
import { LANGUAGES } from '../lang-parser/languages';
import { SupportedLanguage, Block } from '../types';
// ===== 类型定义 =====
/**
* 语言检测配置
* 语言检测配置选项
*/
interface LanguageDetectionConfig {
/** 最小内容长度阈值 */
minContentLength: number;
/** 变化阈值比例 */
changeThreshold: number;
/** 检测置信度阈值 */
confidenceThreshold: number;
/** 空闲检测延迟 (ms) */
idleDelay: number;
/** 默认语言 */
defaultLanguage: SupportedLanguage;
export interface LanguageDetectionConfig {
minContentLength?: number;
confidenceThreshold?: number;
idleDelay?: number;
defaultLanguage?: SupportedLanguage;
}
/**
* 语言检测结果
*/
export interface LanguageDetectionResult {
language: SupportedLanguage;
confidence: number;
}
/**
* Worker 消息接口
*/
interface WorkerMessage {
content: string;
idx: number;
}
/**
* Worker 响应接口
*/
interface WorkerResponse {
language: string;
confidence: number;
idx: number;
}
// ===== 常量配置 =====
/**
* 默认配置
*/
const DEFAULT_CONFIG: LanguageDetectionConfig = {
minContentLength: 8,
changeThreshold: 0.1,
confidenceThreshold: 0.3,
const DEFAULT_CONFIG = {
minContentLength: 20,
confidenceThreshold: 0.15,
idleDelay: 1000,
defaultLanguage: 'text',
defaultLanguage: 'text' as SupportedLanguage,
};
/**
* 语言标记映射
* 将检测结果映射到我们的语言标记
* 支持的语言列表
*/
const DETECTION_TO_TOKEN = Object.fromEntries(
LANGUAGES.map(l => [l.token, l.token])
);
const SUPPORTED_LANGUAGES = new Set([
"json", "py", "html", "sql", "md", "java", "php", "css", "xml",
"cpp", "rs", "cs", "rb", "sh", "yaml", "toml", "go", "clj",
"ex", "erl", "js", "ts", "swift", "kt", "groovy", "ps1", "dart", "scala"
]);
/**
* 兼容性函数
* 语言标记映射表
*/
function requestIdleCallbackCompat(cb: () => void): number {
const LANGUAGE_MAP = new Map(LANGUAGES.map(lang => [lang.token, lang.token]));
// ===== 工具函数 =====
/**
* 兼容性函数：requestIdleCallback
*/
function requestIdleCallbackCompat(callback: () => void): number {
if (typeof window !== 'undefined' && window.requestIdleCallback) {
return window.requestIdleCallback(cb);
} else {
return setTimeout(cb, 0) as any;
return window.requestIdleCallback(callback);
}
return setTimeout(callback, 0) as any;
}
/**
* 兼容性函数：cancelIdleCallback
*/
function cancelIdleCallbackCompat(id: number): void {
if (typeof window !== 'undefined' && window.cancelIdleCallback) {
window.cancelIdleCallback(id);
@@ -71,167 +101,192 @@ function cancelIdleCallbackCompat(id: number): void {
const languageChangeAnnotation = Annotation.define<boolean>();
/**
* 语言更改命令
* 简化版本，直接更新块的语言标记
* 更新代码块语言
*/
function changeLanguageTo(
function updateBlockLanguage(
state: EditorState,
dispatch: (tr: any) => void,
dispatch: (transaction: any) => void,
block: Block,
newLanguage: SupportedLanguage,
isAuto: boolean
newLanguage: SupportedLanguage
): void {
// 构建新的分隔符文本
const autoSuffix = isAuto ? '-a' : '';
const newDelimiter = `\n∞∞∞${newLanguage}${autoSuffix}\n`;
// 创建事务来替换分隔符
const newDelimiter = `\n∞∞∞${newLanguage}-a\n`;
const transaction = state.update({
changes: {
from: block.delimiter.from,
to: block.delimiter.to,
insert: newDelimiter,
},
annotations: [
languageChangeAnnotation.of(true)
]
annotations: [languageChangeAnnotation.of(true)]
});
dispatch(transaction);
}
// ===== Web Worker 管理器 =====
/**
* 语言检测 Worker 管理器
* 负责 Worker 的生命周期管理和消息通信
*/
class LanguageDetectionWorker {
private worker: Worker | null = null;
private pendingRequests = new Map<number, {
resolve: (result: LanguageDetectionResult) => void;
reject: (error: Error) => void;
}>();
private requestId = 0;
constructor() {
this.initWorker();
}
/**
* 初始化 Worker
*/
private initWorker(): void {
try {
this.worker = new Worker('/langdetect-worker.js');
this.worker.onmessage = (event) => {
const response: WorkerResponse = event.data;
const request = this.pendingRequests.get(response.idx);
if (request) {
this.pendingRequests.delete(response.idx);
if (response.language) {
request.resolve({
language: response.language as SupportedLanguage,
confidence: response.confidence
});
} else {
request.reject(new Error('No detection result'));
}
}
};
this.worker.onerror = () => {
this.pendingRequests.forEach(request => request.reject(new Error('Worker error')));
this.pendingRequests.clear();
};
} catch (error) {
console.error('Failed to initialize worker:', error);
}
}
/**
* 检测语言
*/
async detectLanguage(content: string): Promise<LanguageDetectionResult> {
if (!this.worker) {
throw new Error('Worker not initialized');
}
return new Promise((resolve, reject) => {
const id = ++this.requestId;
this.pendingRequests.set(id, { resolve, reject });
this.worker!.postMessage({ content, idx: id } as WorkerMessage);
// 5秒超时
setTimeout(() => {
if (this.pendingRequests.has(id)) {
this.pendingRequests.delete(id);
reject(new Error('Detection timeout'));
}
}, 5000);
});
}
/**
* 销毁 Worker
*/
destroy(): void {
if (this.worker) {
this.worker.terminate();
this.worker = null;
}
this.pendingRequests.clear();
}
}
// ===== 语言检测插件 =====
/**
* 创建语言检测插件
*/
export function createLanguageDetection(
config: Partial<LanguageDetectionConfig> = {}
): ViewPlugin<any> {
export function createLanguageDetection(config: LanguageDetectionConfig = {}): ViewPlugin<any> {
const finalConfig = { ...DEFAULT_CONFIG, ...config };
const previousBlockContent: Record<number, string> = {};
const contentCache = new Map<number, string>();
let idleCallbackId: number | null = null;
let worker: LanguageDetectionWorker | null = null;
return ViewPlugin.fromClass(
class LanguageDetectionPlugin {
constructor(public view: EditorView) {}
constructor(public view: EditorView) {
worker = new LanguageDetectionWorker();
}
update(update: any) {
if (update.docChanged) {
// 取消之前的检测
if (update.docChanged && !update.transactions.some((tr: any) =>
tr.annotation(languageChangeAnnotation))) {
if (idleCallbackId !== null) {
cancelIdleCallbackCompat(idleCallbackId);
idleCallbackId = null;
}
// 安排新的检测
idleCallbackId = requestIdleCallbackCompat(() => {
idleCallbackId = null;
this.performLanguageDetection(update);
this.performDetection(update.state);
});
}
}
private performLanguageDetection(update: any) {
const range = update.state.selection.asSingle().ranges[0];
const blocks = update.state.field(blockState);
private performDetection(state: EditorState): void {
const selection = state.selection.asSingle().ranges[0];
const blocks = state.field(blockState);
let block: Block | null = null;
let blockIndex: number | null = null;
const block = blocks.find(b =>
b.content.from <= selection.from && b.content.to >= selection.from
);
if (!block || !block.language.auto) return;
// 找到当前块
for (let i = 0; i < blocks.length; i++) {
if (blocks[i].content.from <= range.from && blocks[i].content.to >= range.from) {
block = blocks[i];
blockIndex = i;
break;
}
}
const blockIndex = blocks.indexOf(block);
const content = state.doc.sliceString(block.content.from, block.content.to);
if (block === null || blockIndex === null) {
return;
}
// 如果不是自动检测模式，清除缓存并返回
if (!block.language.auto) {
delete previousBlockContent[blockIndex];
return;
}
const content = update.state.doc.sliceString(block.content.from, block.content.to);
// 如果内容为空，重置为默认语言
// 内容为空时重置为默认语言
if (content === "") {
if (block.language.name !== finalConfig.defaultLanguage) {
changeLanguageTo(
update.state,
this.view.dispatch,
block,
finalConfig.defaultLanguage,
true
);
updateBlockLanguage(state, this.view.dispatch, block, finalConfig.defaultLanguage);
}
delete previousBlockContent[blockIndex];
contentCache.delete(blockIndex);
return;
}
// 内容太短，跳过检测
if (content.length <= finalConfig.minContentLength) {
// 内容太短则跳过
if (content.length <= finalConfig.minContentLength) return;
// 检查内容变化
const cachedContent = contentCache.get(blockIndex);
if (cachedContent && levenshteinDistance(cachedContent, content) < content.length * 0.1) {
return;
}
// 检查内容是否有显著变化
const threshold = content.length * finalConfig.changeThreshold;
const previousContent = previousBlockContent[blockIndex];
if (!previousContent) {
// 执行语言检测
this.detectAndUpdateLanguage(content, block, blockIndex, update.state);
previousBlockContent[blockIndex] = content;
} else {
const distance = levenshteinDistance(previousContent, content);
if (distance >= threshold) {
// 执行语言检测
this.detectAndUpdateLanguage(content, block, blockIndex, update.state);
previousBlockContent[blockIndex] = content;
}
}
this.detectAndUpdate(content, block, blockIndex, state);
}
private detectAndUpdateLanguage(
content: string,
block: any,
blockIndex: number,
state: EditorState
) {
private async detectAndUpdate(content: string, block: Block, blockIndex: number, state: EditorState): Promise<void> {
if (!worker) return;
// 使用启发式检测
const detectionResult = detectLanguageHeuristic(content);
// 检查置信度和语言变化
if (detectionResult.confidence >= finalConfig.confidenceThreshold &&
detectionResult.language !== block.language.name &&
DETECTION_TO_TOKEN[detectionResult.language]) {
try {
const result = await worker.detectLanguage(content);
// 验证内容未显著变化
const currentContent = state.doc.sliceString(block.content.from, block.content.to);
const threshold = currentContent.length * finalConfig.changeThreshold;
const contentDistance = levenshteinDistance(content, currentContent);
if (contentDistance <= threshold) {
// 内容未显著变化，安全更新语言
changeLanguageTo(
state,
this.view.dispatch,
block,
detectionResult.language,
true
);
if (result.confidence >= finalConfig.confidenceThreshold &&
result.language !== block.language.name &&
SUPPORTED_LANGUAGES.has(result.language) &&
LANGUAGE_MAP.has(result.language)) {
updateBlockLanguage(state, this.view.dispatch, block, result.language);
}
contentCache.set(blockIndex, content);
} catch (error) {
console.warn('Language detection failed:', error);
}
}
@@ -239,21 +294,38 @@ export function createLanguageDetection(
if (idleCallbackId !== null) {
cancelIdleCallbackCompat(idleCallbackId);
}
if (worker) {
worker.destroy();
worker = null;
}
contentCache.clear();
}
}
);
}
// ===== 公共 API =====
/**
* 手动检测语言
* 手动检测单个内容的语言
*/
export function detectLanguage(content: string): LanguageDetectionResult {
return detectLanguageHeuristic(content);
export async function detectLanguage(content: string): Promise<LanguageDetectionResult> {
const worker = new LanguageDetectionWorker();
try {
return await worker.detectLanguage(content);
} finally {
worker.destroy();
}
}
/**
* 批量检测多个内容块的语言
* 批量检测多个内容的语言
*/
export function detectLanguages(contents: string[]): LanguageDetectionResult[] {
return contents.map(content => detectLanguageHeuristic(content));
export async function detectLanguages(contents: string[]): Promise<LanguageDetectionResult[]> {
const worker = new LanguageDetectionWorker();
try {
return await Promise.all(contents.map(content => worker.detectLanguage(content)));
} finally {
worker.destroy();
}
}

269
frontend/src/views/editor/extensions/codeblock/language-detection/heuristics.ts
View File

@@ -1,269 +0,0 @@
/**
* 基于启发式规则的语言检测
* 用于快速识别常见的编程语言模式
*/
import { SupportedLanguage } from '../types';
/**
* 语言检测结果
*/
export interface LanguageDetectionResult {
language: SupportedLanguage;
confidence: number;
}
/**
* 语言模式定义
*/
interface LanguagePattern {
patterns: RegExp[];
weight: number;
}
/**
* 语言检测规则映射
*/
const LANGUAGE_PATTERNS: Record<string, LanguagePattern> = {
javascript: {
patterns: [
/\b(function|const|let|var|class|extends|import|export|async|await)\b/g,
/\b(console\.log|document\.|window\.)\b/g,
/=>\s*[{(]/g,
/\b(require|module\.exports)\b/g,
],
weight: 1.0,
},
typescript: {
patterns: [
/\b(interface|type|enum|namespace|implements|declare)\b/g,
/:\s*(string|number|boolean|object|any)\b/g,
/<[A-Z][a-zA-Z0-9<>,\s]*>/g,
/\b(public|private|protected|readonly)\b/g,
],
weight: 1.2,
},
python: {
patterns: [
/\b(def|class|import|from|if __name__|print|len|range)\b/g,
/^\s*#.*$/gm,
/\b(True|False|None)\b/g,
/:\s*$/gm,
],
weight: 1.0,
},
java: {
patterns: [
/\b(public|private|protected|static|final|class|interface)\b/g,
/\b(System\.out\.println|String|int|void)\b/g,
/import\s+[a-zA-Z0-9_.]+;/g,
/\b(extends|implements)\b/g,
],
weight: 1.0,
},
html: {
patterns: [
/<\/?[a-zA-Z][^>]*>/g,
/<!DOCTYPE\s+html>/gi,
/<(div|span|p|h[1-6]|body|head|html)\b/g,
/\s(class|id|src|href)=/g,
],
weight: 1.5,
},
css: {
patterns: [
/[.#][a-zA-Z][\w-]*\s*{/g,
/\b(color|background|margin|padding|font-size):\s*[^;]+;/g,
/@(media|keyframes|import)\b/g,
/\{[^}]*\}/g,
],
weight: 1.3,
},
json: {
patterns: [
/^\s*[{\[][\s\S]*[}\]]\s*$/,
/"[^"]*":\s*(".*"|[\d.]+|true|false|null)/g,
/,\s*$/gm,
],
weight: 2.0,
},
sql: {
patterns: [
/\b(SELECT|FROM|WHERE|INSERT|UPDATE|DELETE|CREATE|ALTER|DROP)\b/gi,
/\b(JOIN|LEFT|RIGHT|INNER|OUTER|ON|GROUP BY|ORDER BY)\b/gi,
/;\s*$/gm,
/\b(TABLE|DATABASE|INDEX)\b/gi,
],
weight: 1.4,
},
shell: {
patterns: [
/^#!/g,
/\b(echo|cd|ls|grep|awk|sed|cat|chmod)\b/g,
/\$\{?\w+\}?/g,
/\|\s*\w+/g,
],
weight: 1.2,
},
markdown: {
patterns: [
/^#+\s+/gm,
/\*\*.*?\*\*/g,
/\[.*?\]\(.*?\)/g,
/^```/gm,
],
weight: 1.1,
},
php: {
patterns: [
/<\?php/g,
/\$\w+/g,
/\b(function|class|extends|implements)\b/g,
/echo\s+/g,
],
weight: 1.3,
},
cpp: {
patterns: [
/#include\s*<.*>/g,
/\b(int|char|float|double|void|class|struct)\b/g,
/std::/g,
/cout\s*<<|cin\s*>>/g,
],
weight: 1.1,
},
rust: {
patterns: [
/\bfn\s+\w+/g,
/\b(let|mut|struct|enum|impl|trait)\b/g,
/println!\(/g,
/::\w+/g,
],
weight: 1.2,
},
go: {
patterns: [
/\bfunc\s+\w+/g,
/\b(var|const|type|package|import)\b/g,
/fmt\.\w+/g,
/:=\s*/g,
],
weight: 1.1,
},
ruby: {
patterns: [
/\b(def|class|module|end)\b/g,
/\b(puts|print|require)\b/g,
/@\w+/g,
/\|\w+\|/g,
],
weight: 1.0,
},
yaml: {
patterns: [
/^\s*\w+:\s*.*$/gm, // key: value 模式
/^\s*-\s+\w+/gm, // 列表项
/^---\s*$/gm, // 文档分隔符
/^\s*\w+:\s*\|/gm, // 多行字符串
/^\s*\w+:\s*>/gm, // 折叠字符串
/^\s*#.*$/gm, // 注释
/:\s*\[.*\]/g, // 内联数组
/:\s*\{.*\}/g, // 内联对象
],
weight: 1.5,
},
xml: {
patterns: [
/<\?xml/g,
/<\/\w+>/g,
/<\w+[^>]*\/>/g,
/\s\w+="[^"]*"/g,
],
weight: 1.3,
},
};
/**
* JSON 特殊检测
* 使用更严格的规则检测 JSON
*/
function detectJSON(content: string): LanguageDetectionResult | null {
const trimmed = content.trim();
if ((trimmed.startsWith('{') && trimmed.endsWith('}')) ||
(trimmed.startsWith('[') && trimmed.endsWith(']'))) {
try {
JSON.parse(trimmed);
return {
language: 'json',
confidence: 1.0,
};
} catch (e) {
// JSON 解析失败，继续其他检测
}
}
return null;
}
/**
* 计算文本与语言模式的匹配分数
*/
function calculateScore(content: string, pattern: LanguagePattern): number {
let score = 0;
const contentLength = Math.max(content.length, 1);
for (const regex of pattern.patterns) {
const matches = content.match(regex);
if (matches) {
score += matches.length;
}
}
// 根据内容长度和权重标准化分数
return (score * pattern.weight) / (contentLength / 100);
}
/**
* 基于启发式规则检测语言
*/
export function detectLanguageHeuristic(content: string): LanguageDetectionResult {
if (!content.trim()) {
return { language: 'text', confidence: 1.0 };
}
// 首先尝试 JSON 特殊检测
const jsonResult = detectJSON(content);
if (jsonResult) {
return jsonResult;
}
const scores: Record<string, number> = {};
// 计算每种语言的匹配分数
for (const [language, pattern] of Object.entries(LANGUAGE_PATTERNS)) {
scores[language] = calculateScore(content, pattern);
}
// 找到最高分的语言
const sortedScores = Object.entries(scores)
.sort(([, a], [, b]) => b - a)
.filter(([, score]) => score > 0);
if (sortedScores.length > 0) {
const [bestLanguage, bestScore] = sortedScores[0];
return {
language: bestLanguage as SupportedLanguage,
confidence: Math.min(bestScore, 1.0),
};
}
return { language: 'text', confidence: 1.0 };
}
/**
* 获取所有支持的检测语言
*/
export function getSupportedDetectionLanguages(): SupportedLanguage[] {
return Object.keys(LANGUAGE_PATTERNS) as SupportedLanguage[];
}

24
frontend/src/views/editor/extensions/codeblock/language-detection/index.ts
View File

@@ -1,26 +1,14 @@
/**
* 语言检测模块入口
* 导出所有语言检测相关的功能
* 语言检测模块
*/
// 主要检测功能
export {
createLanguageDetection,
detectLanguage,
detectLanguages
detectLanguages,
type LanguageDetectionResult,
type LanguageDetectionConfig
} from './autodetect';
// 启发式检测
export {
detectLanguageHeuristic,
getSupportedDetectionLanguages,
type LanguageDetectionResult
} from './heuristics';
// 工具函数
export {
levenshteinDistance
} from './levenshtein';
// 重新导出类型
export type { SupportedLanguage } from '../types';
export { levenshteinDistance } from './levenshtein';
export type { SupportedLanguage } from '../types';

86
frontend/src/views/editor/extensions/codeblock/language-detection/levenshtein.ts
View File

@@ -1,78 +1,70 @@
/**
* Levenshtein 距离算法
* 用于计算两个字符串之间的编辑距离
*/
/**
* 内部最小值计算函数
* 用于计算编辑距离的最小成本
*
* @param d0 - 对角线上的距离
* @param d1 - 左侧的距离
* @param d2 - 上方的距离
* @param bx - 字符串 b 的当前字符
* @param ay - 字符串 a 的当前字符
* @returns 最小编辑距离
*/
function _min(d0: number, d1: number, d2: number, bx: number, ay: number): number {
function min(d0: number, d1: number, d2: number, bx: number, ay: number): number {
return d0 < d1 || d2 < d1
? d0 > d2
? d2 + 1
: d0 + 1
: bx === ay
? d1
: d1 + 1;
? d0 > d2 ? d2 + 1 : d0 + 1
: bx === ay ? d1 : d1 + 1;
}
/**
* 计算两个字符串之间的 Levenshtein 距离
* @param a 第一个字符串
* @param b 第二个字符串
* @returns 编辑距离
*
* 该实现使用了多项优化：
* 1. 确保较短的字符串作为第一个参数
* 2. 跳过公共前缀和后缀
* 3. 使用滚动数组减少空间复杂度
* 4. 批量处理以提高性能
*
* @param stringA - 第一个字符串
* @param stringB - 第二个字符串
* @returns 编辑距离（非负整数）
*/
export function levenshteinDistance(a: string, b: string): number {
if (a === b) {
return 0;
}
if (a === b) return 0;
if (a.length > b.length) {
const tmp = a;
a = b;
b = tmp;
[a, b] = [b, a];
}
let la = a.length;
let lb = b.length;
while (la > 0 && (a.charCodeAt(la - 1) === b.charCodeAt(lb - 1))) {
// 跳过公共后缀
while (la > 0 && a.charCodeAt(la - 1) === b.charCodeAt(lb - 1)) {
la--;
lb--;
}
let offset = 0;
while (offset < la && (a.charCodeAt(offset) === b.charCodeAt(offset))) {
// 跳过公共前缀
while (offset < la && a.charCodeAt(offset) === b.charCodeAt(offset)) {
offset++;
}
la -= offset;
lb -= offset;
if (la === 0 || lb < 3) {
return lb;
}
if (la === 0 || lb < 3) return lb;
let x = 0;
let y: number;
let d0: number;
let d1: number;
let d2: number;
let d3: number;
let dd = 0;
let dy: number;
let ay: number;
let bx0: number;
let bx1: number;
let bx2: number;
let bx3: number;
let x = 0, y: number, d0: number, d1: number, d2: number, d3: number;
let dd = 0, dy: number, ay: number, bx0: number, bx1: number, bx2: number, bx3: number;
const vector: number[] = [];
for (y = 0; y < la; y++) {
vector.push(y + 1);
vector.push(a.charCodeAt(offset + y));
vector.push(y + 1, a.charCodeAt(offset + y));
}
const len = vector.length - 1;
@@ -84,18 +76,16 @@ export function levenshteinDistance(a: string, b: string): number {
bx3 = b.charCodeAt(offset + (d3 = x + 3));
x += 4;
dd = x;
for (y = 0; y < len; y += 2) {
dy = vector[y];
ay = vector[y + 1];
d0 = _min(dy, d0, d1, bx0, ay);
d1 = _min(d0, d1, d2, bx1, ay);
d2 = _min(d1, d2, d3, bx2, ay);
dd = _min(d2, d3, dd, bx3, ay);
d0 = min(dy, d0, d1, bx0, ay);
d1 = min(d0, d1, d2, bx1, ay);
d2 = min(d1, d2, d3, bx2, ay);
dd = min(d2, d3, dd, bx3, ay);
vector[y] = dd;
d3 = d2;
d2 = d1;
d1 = d0;
d0 = dy;
d3 = d2; d2 = d1; d1 = d0; d0 = dy;
}
}
@@ -104,7 +94,7 @@ export function levenshteinDistance(a: string, b: string): number {
dd = ++x;
for (y = 0; y < len; y += 2) {
dy = vector[y];
vector[y] = dd = _min(dy, d0, dd, bx0, vector[y + 1]);
vector[y] = dd = min(dy, d0, dd, bx0, vector[y + 1]);
d0 = dy;
}
}

77
frontend/src/views/editor/extensions/codeblock/types.ts
View File

@@ -25,48 +25,68 @@ export interface Block {
*/
export type SupportedLanguage =
| 'text'
| 'javascript'
| 'typescript'
| 'python'
| 'html'
| 'css'
| 'json'
| 'markdown'
| 'shell'
| 'py' // Python
| 'html'
| 'sql'
| 'yaml'
| 'xml'
| 'php'
| 'md' // Markdown
| 'java'
| 'cpp'
| 'c'
| 'php'
| 'css'
| 'xml'
| 'cpp' // C++
| 'rs' // Rust
| 'cs' // C#
| 'rb' // Ruby
| 'sh' // Shell
| 'yaml'
| 'toml'
| 'go'
| 'rust'
| 'ruby';
| 'clj' // Clojure
| 'ex' // Elixir
| 'erl' // Erlang
| 'js' // JavaScript
| 'ts' // TypeScript
| 'swift'
| 'kt' // Kotlin
| 'groovy'
| 'ps1' // PowerShell
| 'dart'
| 'scala';
/**
* 支持的语言列表
*/
export const SUPPORTED_LANGUAGES: SupportedLanguage[] = [
'text',
'javascript',
'typescript',
'python',
'html',
'css',
'json',
'markdown',
'shell',
'py',
'html',
'sql',
'yaml',
'xml',
'php',
'md',
'java',
'php',
'css',
'xml',
'cpp',
'c',
'rs',
'cs',
'rb',
'sh',
'yaml',
'toml',
'go',
'rust',
'ruby'
'clj',
'ex',
'erl',
'js',
'ts',
'swift',
'kt',
'groovy',
'ps1',
'dart',
'scala'
];
/**
@@ -134,9 +154,6 @@ export interface BlockStateUpdate {
operation?: BlockOperation;
}
// 块导航方向
export type NavigationDirection = 'next' | 'previous' | 'first' | 'last';
// 语言检测结果
export interface LanguageDetectionResult {
language: SupportedLanguage;