repo2txt / js /index.js
multimodalart's picture
Update js/index.js
f158af3 verified
import { displayDirectoryStructure, getSelectedFiles, formatRepoContents } from './utils.js';
let currentRepoInfo = {};
// Load saved token on page load
document.addEventListener('DOMContentLoaded', function() {
lucide.createIcons();
setupShowMoreInfoButton();
loadSavedToken();
});
// Load saved token from local storage
function loadSavedToken() {
const savedToken = localStorage.getItem('githubAccessToken');
if (savedToken) {
document.getElementById('accessToken').value = savedToken;
}
}
// Save token to local storage
function saveToken(token) {
if (token) {
localStorage.setItem('githubAccessToken', token);
} else {
localStorage.removeItem('githubAccessToken');
}
}
// Event listener for form submission
document.getElementById('repoForm').addEventListener('submit', async function (e) {
e.preventDefault();
const repoUrl = document.getElementById('repoUrl').value;
const accessToken = document.getElementById('accessToken').value;
saveToken(accessToken);
const outputText = document.getElementById('outputText');
outputText.value = 'Fetching repository structure...';
document.getElementById('directoryStructure').innerHTML = '';
document.getElementById('generateTextButton').style.display = 'none';
document.getElementById('downloadZipButton').style.display = 'none';
document.getElementById('copyButton').style.display = 'none';
document.getElementById('downloadButton').style.display = 'none';
try {
const repoInfo = parseRepoUrl(repoUrl);
currentRepoInfo = { ...repoInfo, accessToken }; // Store for later use
let tree;
if (repoInfo.source === 'github') {
const { owner, repo, lastString } = repoInfo;
let refFromUrl = '';
let pathFromUrl = '';
if (lastString) {
const references = await getGitHubReferences(owner, repo, accessToken);
const allRefs = [...references.branches, ...references.tags];
const matchingRef = allRefs.find(ref => lastString.startsWith(ref));
if (matchingRef) {
refFromUrl = matchingRef;
pathFromUrl = lastString.slice(matchingRef.length + 1);
} else {
refFromUrl = lastString;
}
}
const sha = await fetchRepoSha(owner, repo, refFromUrl, pathFromUrl, accessToken);
tree = await fetchGitHubRepoTree(owner, repo, sha, accessToken);
} else if (repoInfo.source === 'huggingface') {
const { owner, repo, repo_type, lastString } = repoInfo;
let refFromUrl = 'main'; // Default branch
let pathFromUrl = '';
if (lastString) {
const refs = await getHuggingFaceReferences(owner, repo, repo_type, accessToken);
const matchingRef = refs.find(ref => lastString.startsWith(ref + '/'));
if (matchingRef) {
refFromUrl = matchingRef;
pathFromUrl = lastString.slice(matchingRef.length + 1);
} else {
refFromUrl = lastString.split('/')[0];
pathFromUrl = lastString.substring(refFromUrl.length + 1);
}
}
currentRepoInfo.ref = refFromUrl;
tree = await fetchHuggingFaceTree(owner, repo, repo_type, refFromUrl, pathFromUrl, accessToken);
}
displayDirectoryStructure(tree);
document.getElementById('generateTextButton').style.display = 'flex';
document.getElementById('downloadZipButton').style.display = 'flex';
outputText.value = 'Select files and click "Generate Text File" or "Download Zip".';
} catch (error) {
outputText.value = `Error fetching repository contents: ${error.message}\n\n` +
"Please ensure:\n" +
"1. The repository URL is correct and accessible.\n" +
"2. You have the necessary permissions to access the repository.\n" +
"3. If it's a private repository, you've provided a valid access token.\n" +
"4. The specified branch/tag and path (if any) exist in the repository.";
}
});
// Event listener for generating text file
document.getElementById('generateTextButton').addEventListener('click', async function () {
const accessToken = document.getElementById('accessToken').value;
const outputText = document.getElementById('outputText');
outputText.value = 'Generating text file...';
saveToken(accessToken);
try {
const selectedFiles = getSelectedFiles();
if (selectedFiles.length === 0) {
throw new Error('No files selected');
}
const fileContents = await fetchFileContents(selectedFiles, accessToken, currentRepoInfo.source);
const formattedText = formatRepoContents(fileContents);
outputText.value = formattedText;
document.getElementById('copyButton').style.display = 'flex';
document.getElementById('downloadButton').style.display = 'flex';
} catch (error) {
outputText.value = `Error generating text file: ${error.message}\n\n` +
"Please ensure:\n" +
"1. You have selected at least one file from the directory structure.\n" +
"2. Your access token (if provided) is valid and has the necessary permissions.\n" +
"3. You have a stable internet connection.";
}
});
// Event listener for downloading zip file
document.getElementById('downloadZipButton').addEventListener('click', async function () {
const accessToken = document.getElementById('accessToken').value;
try {
const selectedFiles = getSelectedFiles();
if (selectedFiles.length === 0) {
throw new Error('No files selected');
}
const fileContents = await fetchFileContents(selectedFiles, accessToken, currentRepoInfo.source);
await createAndDownloadZip(fileContents);
} catch (error) {
const outputText = document.getElementById('outputText');
outputText.value = `Error generating zip file: ${error.message}\n\n` +
"Please ensure:\n" +
"1. You have selected at least one file from the directory structure.\n" +
"2. Your access token (if provided) is valid and has the necessary permissions.";
}
});
// Event listener for copying text to clipboard
document.getElementById('copyButton').addEventListener('click', function () {
const outputText = document.getElementById('outputText');
outputText.select();
navigator.clipboard.writeText(outputText.value)
.then(() => console.log('Text copied to clipboard'))
.catch(err => console.error('Failed to copy text: ', err));
});
// Event listener for downloading text file
document.getElementById('downloadButton').addEventListener('click', function () {
const outputText = document.getElementById('outputText').value;
if (!outputText.trim()) {
document.getElementById('outputText').value = 'Error: No content to download. Please generate the text file first.';
return;
}
const blob = new Blob([outputText], { type: 'text/plain' });
const url = URL.createObjectURL(blob);
const a = document.createElement('a');
a.href = url;
a.download = 'prompt.txt';
a.click();
URL.revokeObjectURL(url);
});
// Parse GitHub or Hugging Face repository URL
function parseRepoUrl(url) {
url = url.replace(/\/$/, '');
const githubPattern = /^https:\/\/github\.com\/([^\/]+)\/([^\/]+)(?:\/tree\/(.+))?$/;
const hfPattern = /^https:\/\/huggingface\.co\/(?:(datasets|spaces)\/)?([^\/]+)\/([^\/]+)(?:\/tree\/(.+))?$/;
let match = url.match(githubPattern);
if (match) {
return {
source: 'github',
owner: match[1],
repo: match[2],
lastString: match[4] || ''
};
}
match = url.match(hfPattern);
if (match) {
let repo_type = 'model';
if (match[1] === 'datasets') repo_type = 'dataset';
if (match[1] === 'spaces') repo_type = 'space';
return {
source: 'huggingface',
repo_type: repo_type,
owner: match[2],
repo: match[3],
lastString: match[4] || ''
};
}
throw new Error('Invalid GitHub or Hugging Face repository URL.');
}
// Fetch GitHub repository references
async function getGitHubReferences(owner, repo, token) {
const headers = { 'Accept': 'application/vnd.github+json' };
if (token) headers['Authorization'] = `token ${token}`;
const [branchesResponse, tagsResponse] = await Promise.all([
fetch(`https://api.github.com/repos/${owner}/${repo}/branches`, { headers }),
fetch(`https://api.github.com/repos/${owner}/${repo}/tags`, { headers })
]);
if (!branchesResponse.ok || !tagsResponse.ok) handleFetchError(branchesResponse, 'github');
const branches = await branchesResponse.json();
const tags = await tagsResponse.json();
return {
branches: branches.map(b => b.name),
tags: tags.map(t => t.name)
};
}
// Fetch Hugging Face repository references
async function getHuggingFaceReferences(owner, repo, repo_type, token) {
const repoId = `${owner}/${repo}`;
const typePath = repo_type === 'model' ? 'models' : repo_type === 'dataset' ? 'datasets' : 'spaces';
const url = `https://huggingface.co/api/${typePath}/${repoId}/refs`;
const headers = {};
if (token) headers['Authorization'] = `Bearer ${token}`;
const response = await fetch(url, { headers });
if (!response.ok) handleFetchError(response, 'huggingface');
const data = await response.json();
const branches = data.branches ? data.branches.map(b => b.name) : [];
const tags = data.tags ? data.tags.map(t => t.name) : [];
return [...branches, ...tags];
}
// Fetch repository SHA
async function fetchRepoSha(owner, repo, ref, path, token) {
const url = `https://api.github.com/repos/${owner}/${repo}/contents/${path ? `${path}` : ''}${ref ? `?ref=${ref}` : ''}`;
const headers = { 'Accept': 'application/vnd.github.object+json' };
if (token) headers['Authorization'] = `token ${token}`;
const response = await fetch(url, { headers });
if (!response.ok) handleFetchError(response, 'github');
const data = await response.json();
return data.sha;
}
// Fetch GitHub repository tree
async function fetchGitHubRepoTree(owner, repo, sha, token) {
const url = `https://api.github.com/repos/${owner}/${repo}/git/trees/${sha}?recursive=1`;
const headers = { 'Accept': 'application/vnd.github+json' };
if (token) headers['Authorization'] = `token ${token}`;
const response = await fetch(url, { headers });
if (!response.ok) handleFetchError(response, 'github');
const data = await response.json();
return data.tree;
}
// Fetch Hugging Face repository tree
async function fetchHuggingFaceTree(owner, repo, repo_type, ref, path, token) {
const typePath = repo_type === 'model' ? 'models' : repo_type === 'dataset' ? 'datasets' : 'spaces';
const url = `https://huggingface.co/api/${typePath}/${owner}/${repo}/tree/${ref}?recursive=true`;
const headers = {};
if (token) headers['Authorization'] = `Bearer ${token}`;
const response = await fetch(url, { headers });
if (!response.ok) handleFetchError(response, 'huggingface');
let tree = await response.json();
if (path) {
tree = tree.filter(item => item.path.startsWith(path + '/') || item.path === path);
}
return tree.map(item => {
let repoIdForUrl;
switch (repo_type) {
case 'dataset':
repoIdForUrl = `datasets/${owner}/${repo}`;
break;
case 'space':
repoIdForUrl = `spaces/${owner}/${repo}`;
break;
default: // model
repoIdForUrl = `${owner}/${repo}`;
}
return {
path: item.path,
type: (item.type === 'file' || item.type === 'lfs') ? 'blob' : 'tree',
urlType: 'hf',
url: `https://huggingface.co/${repoIdForUrl}/raw/${ref}/${item.path}`
};
});
}
// Handle fetch errors
function handleFetchError(response, source = 'github') {
if (response.status === 403 && source === 'github' && response.headers.get('X-RateLimit-Remaining') === '0') {
throw new Error('GitHub API rate limit exceeded. Please try again later or provide a valid access token to increase your rate limit.');
}
if (response.status === 401) {
throw new Error(`Authentication error. Please check if your access token is valid and has the required permissions.`);
}
if (response.status === 404) {
throw new Error(`Repository, branch, or path not found. Please check that the URL, branch/tag, and path are correct and accessible.`);
}
throw new Error(`Failed to fetch repository data. Status: ${response.status}. Please check your input and try again.`);
}
// Fetch contents of selected files
async function fetchFileContents(files, token, source) {
const contents = await Promise.all(files.map(async file => {
let headers = {};
if (token) {
headers['Authorization'] = source === 'github' ? `token ${token}` : `Bearer ${token}`;
}
if (source === 'github') {
headers['Accept'] = 'application/vnd.github.v3.raw';
}
const response = await fetch(file.url, { headers });
if (!response.ok) handleFetchError(response, source);
const text = await response.text();
return { url: file.url, path: file.path, text };
}));
return contents;
}
function setupShowMoreInfoButton() {
const showMoreInfoButton = document.getElementById('showMoreInfo');
const tokenInfo = document.getElementById('tokenInfo');
showMoreInfoButton.addEventListener('click', function() {
tokenInfo.classList.toggle('hidden');
updateInfoIcon(this, tokenInfo);
});
}
function updateInfoIcon(button, tokenInfo) {
const icon = button.querySelector('[data-lucide]');
if (icon) {
icon.setAttribute('data-lucide', tokenInfo.classList.contains('hidden') ? 'info' : 'x');
lucide.createIcons();
}
}
// Create and download zip file
async function createAndDownloadZip(fileContents) {
const zip = new JSZip();
fileContents.forEach(file => {
const filePath = file.path.startsWith('/') ? file.path.slice(1) : file.path;
zip.file(filePath, file.text);
});
const content = await zip.generateAsync({type: "blob"});
const url = URL.createObjectURL(content);
const a = document.createElement('a');
a.href = url;
a.download = 'repo_contents.zip';
document.body.appendChild(a);
a.click();
document.body.removeChild(a);
URL.revokeObjectURL(url);
}