" Vimball Archiver by Charles E. Campbell, Jr., Ph.D. UseVimball finish ftplugin/csv.vim [[[1 978 " Filetype plugin for editing CSV files. "{{{1 " Author: Christian Brabandt " Version: 0.18 " Script: http://www.vim.org/scripts/script.php?script_id=2830 " License: VIM License " Last Change: Tue, 30 Aug 2011 21:42:59 +0200 " Documentation: see :help ft_csv.txt " GetLatestVimScripts: 2830 18 :AutoInstall: csv.vim " " Some ideas are take from the wiki http://vim.wikia.com/wiki/VimTip667 " though, implementation differs. " Plugin folklore "{{{2 if v:version < 700 || exists('b:did_ftplugin') finish endif let b:did_ftplugin = 1 let s:cpo_save = &cpo set cpo&vim " Function definitions: "{{{2 fu! Warn(mess) "{{{3 echohl WarningMsg echomsg "CSV: " . a:mess echohl Normal endfu fu! Init() "{{{3 " Hilight Group for Columns if exists("g:csv_hiGroup") let s:hiGroup = g:csv_hiGroup else let s:hiGroup="WildMenu" endif if !exists("g:csv_hiHeader") let s:hiHeader = "Title" else let s:hiHeader = g:csv_hiHeader endif exe "hi link CSVHeaderLine" s:hiHeader " Determine default Delimiter if !exists("g:csv_delim") let b:delimiter=GetDelimiter() else let b:delimiter=g:csv_delim endif if empty(b:delimiter) && !exists("b:csv_fixed_width") call Warn("No delimiter found. See :h csv-delimiter to set it manually!") endif let s:del='\%(' . b:delimiter . '\|$\)' " Pattern for matching a single column if !exists("g:csv_strict_columns") && !exists("g:csv_col") \ && !exists("b:csv_fixed_width") " - Allow double quotes as escaped quotes only insides double quotes " - Allow linebreaks only, if g:csv_nl isn't set (this is " only allowed in double quoted strings see RFC4180), though this " does not work with :WhatColumn and might mess up syntax " highlighting. " - optionally allow whitespace in front of the fields (to make it " work with :ArrangeCol (that is actually not RFC4180 valid)) " - Should work with most ugly solutions that are available let b:col='\%(\%(\%(' . (b:delimiter !~ '\s' ? '\s*' : '') . \ '"\%(' . (exists("g:csv_nl") ? '\_' : '' ) . \ '[^"]\|""\)*"\)' . s:del . \ '\)\|\%(' . \ '[^' . b:delimiter . ']*' . s:del . '\)\)' elseif !exists("g:csv_col") && exists("g:csv_strict_columns") " strict columns let b:col='\%([^' . b:delimiter . ']*' . s:del . '\)' elseif exists("b:csv_fixed_width") " Fixed width column let b:col='' " Check for sane default if b:csv_fixed_width =~? '[^0-9,]' call Warn("Please specify the list of character columns" . \ "like this: '1,3,5'. See also :h csv-fixedwidth") return endif let b:csv_fixed_width_cols=split(b:csv_fixed_width, ',') " Force evaluating as numbers call map(b:csv_fixed_width_cols, 'v:val+0') else " User given column definition let b:col = g:csv_col endif " define buffer-local commands call CommandDefinitions() " CSV specific mappings call CSVMappings() " Highlight column, on which the cursor is? if exists("g:csv_highlight_column") && g:csv_highlight_column =~? 'y' && \ !exists("#CSV_HI#CursorMoved") aug CSV_HI au! au CursorMoved HiColumn aug end " Set highlighting for column, on which the cursor is currently HiColumn elseif exists("#CSV_HI#CursorMoved") aug CSV_HI au! CursorMoved aug end aug! CSV_HI " Remove any existing highlighting HiColumn! endif " force reloading CSV Syntax Highlighting if exists("b:current_syntax") unlet b:current_syntax " Force reloading syntax file endif if !exists("#CSV#ColorScheme") " Make sure, syntax highlighting is applied " after changing the colorscheme augroup CSV au! au ColorScheme *.csv,*.dat do Syntax augroup end endif silent do Syntax " undo when setting a new filetype let b:undo_ftplugin = "setlocal sol< tw< wrap<" \ . "| unlet! b:delimiter b:col b:csv_fixed_width_cols" " CSV local settings setl nostartofline tw=0 nowrap if has("conceal") setl cole=2 cocu=nc let b:undo_ftplugin .= '|setl cole< cocu<' endif endfu fu! SearchColumn(arg) "{{{3 try let arglist=split(a:arg) if len(arglist) == 1 let colnr=WColumn() let pat=substitute(arglist[0], '^\(.\)\(.*\)\1$', '\2', '') if pat == arglist[0] throw "E684" endif else let colnr=arglist[0] let pat=substitute(arglist[1], '^\(.\)\(.*\)\1$', '\2', '') if pat == arglist[1] throw "E684" endif endif "catch /^Vim\%((\a\+)\)\=:E684/ catch /E684/ " catch error index out of bounds call Warn("Error! Usage :SearchInColumn [] /pattern/") return 1 endtry let maxcolnr = MaxColumns() if colnr > maxcolnr call Warn("There exists no column " . colnr) return 1 endif "let @/=GetColPat(colnr) . '*\zs' . pat . '\ze\([^' . b:delimiter . ']*' . b:delimiter .'\)\?' . GetColPat(maxcolnr-colnr-1) " GetColPat(nr) returns a pattern containing '\zs' if nr > 1, " therefore, we need to clear that flag again ;( " TODO: " Is there a better way, than running a substitute command on '\zs', may be using a flag " with GetColPat(zsflag, colnr)? if colnr > 1 && colnr < maxcolnr "let @/=GetColPat(colnr-1,0) . '*\zs' . pat . '\ze\([^' . b:delimiter . ']*' . b:delimiter .'\)\?' . GetColPat(maxcolnr-colnr-1,0) "let @/= '^' . GetColPat(colnr-1,0) . '[^' . b:delimiter . ']*\zs' . pat . '\ze[^' . b:delimiter . ']*'.b:delimiter . GetColPat(maxcolnr-colnr,0) . '$' "let @/= '^' . GetColPat(colnr-1,0) . b:col1 . '\?\zs' . pat . '\ze' . b:col1 .'\?' . GetColPat(maxcolnr-colnr,0) " . '$' if !exists("b:csv_fixed_width_cols") let @/= '^' . GetColPat(colnr-1,0) . '\%([^' . b:delimiter . \ ']*\)\?\zs' . pat . '\ze' . '\%([^' . b:delimiter .']*\)\?' . \ b:delimiter . GetColPat(maxcolnr-colnr,0) . '$' else let @/ = '\%' . b:csv_fixed_width_cols[(colnr-1)] . 'c\zs' . pat \ . '.\{-}\ze\%' . (b:csv_fixed_width_cols[colnr]) . 'c\ze' endif elseif colnr == maxcolnr if !exists("b:csv_fixed_width_cols") let @/= '^' . GetColPat(colnr-1,0) . '\%([^' . b:delimiter . \ ']*\)\?\zs' . pat . '\ze' else let @/= '\%' . b:csv_fixed_width_cols[-1] . 'c\zs' . pat . '\ze' endif else " colnr = 1 "let @/= '^\zs' . pat . '\ze' . substitute((GetColPat(maxcolnr - colnr)), '\\zs', '', 'g') "let @/= '^\zs' . b:col1 . '\?' . pat . '\ze' . b:col1 . '\?' . GetColPat(maxcolnr,0) . '$' if !exists("b:csv_fixed_width_cols") let @/= '^' . '\%([^' . b:delimiter . ']*\)\?\zs' . pat . \ '\ze\%([^' . b:delimiter . ']*\)\?' . b:delimiter . \ GetColPat(maxcolnr-1,0) . '$' else let @/= pat . '\ze.\{-}\%' . b:csv_fixed_width_cols[1] . 'c' endif endif try norm! n catch /^Vim\%((\a\+)\)\=:E486/ " Pattern not found echohl Error echomsg "E486: Pattern not found in column " . colnr . ": " . pat if &vbs > 0 echomsg substitute(v:exception, '^[^:]*:', '','') endif echohl Normal endtry endfu fu! DelColumn(colnr) "{{{3 let maxcolnr = MaxColumns() let _p = getpos('.') if empty(a:colnr) let colnr=WColumn() else let colnr=a:colnr endif if colnr > maxcolnr call Warn("There exists no column " . colnr) return endif if colnr != '1' if !exists("b:csv_fixed_width_cols") let pat= '^' . GetColPat(colnr-1,1) . b:col else let pat= GetColPat(colnr,0) endif else " distinction between csv and fixed width does not matter here let pat= '^' . GetColPat(colnr,0) endif if &ro setl noro endif exe ':%s/' . escape(pat, '/') . '//' call setpos('.', _p) endfu fu! HiCol(colnr) "{{{3 if a:colnr > MaxColumns() && a:colnr[-1:] != '!' call Warn("There exists no column " . a:colnr) return endif if a:colnr[-1:] != '!' if empty(a:colnr) let colnr=WColumn() else let colnr=a:colnr endif if colnr==1 let pat='^'. GetColPat(colnr,0) else if !exists("b:csv_fixed_width_cols") let pat='^'. GetColPat(colnr-1,1) . b:col else let pat=GetColPat(colnr,0) endif endif endif if exists("*matchadd") if exists("s:matchid") " ignore errors, that come from already deleted matches sil! call matchdelete(s:matchid) endif " Additionally, filter all matches, that could have been used earlier let matchlist=getmatches() call filter(matchlist, 'v:val["group"] !~ s:hiGroup') call setmatches(matchlist) if a:colnr[-1:] == '!' return endif let s:matchid=matchadd(s:hiGroup, pat, 0) else if a:colnr[-1:] != '!' exe ":2match " . s:hiGroup . ' /' . pat . '/' endif endif endfu fu! GetDelimiter() "{{{3 if !exists("b:csv_fixed_width_cols") let _cur = getpos('.') let Delim={0: ';', 1: ',', 2: '|', 3: ' '} let temp={} for i in values(Delim) redir => temp[i] exe "silent! %s/" . i . "/&/nge" redir END endfor let Delim = map(temp, 'matchstr(substitute(v:val, "\n", "", ""), "^\\d\\+")') let result=[] for [key, value] in items(Delim) if get(result,0) < value call add(result, key) call add(result, value) endif endfor call setpos('.', _cur) if !empty(result) return result[0] else return '' endif else " There is no delimiter for fixedwidth files return '' endif endfu fu! WColumn(...) "{{{3 " Return on which column the cursor is let _cur = getpos('.') if !exists("b:csv_fixed_width_cols") let line=getline('.') " move cursor to end of field "call search(b:col, 'ec', line('.')) call search(b:col, 'ec') let end=col('.')-1 let fields=(split(line[0:end],b:col.'\zs')) let ret=len(fields) if exists("a:1") && a:1 > 0 " bang attribute let head = split(getline(1),b:col.'\zs') " remove preceeding whitespace let ret = substitute(head[ret-1], '^\s\+', '', '') " remove delimiter let ret = substitute(ret, b:delimiter. '$', '', '') endif else let temp=getpos('.')[2] let j=1 let ret = 1 for i in sort(b:csv_fixed_width_cols, "SortList") if temp >= i let ret = j endif let j += 1 endfor endif call setpos('.',_cur) return ret endfu fu! MaxColumns() "{{{3 "return maximum number of columns in first 10 lines if !exists("b:csv_fixed_width_cols") let l=getline(1,10) let fields=[] let result=0 for item in l let temp=len(split(item, b:col.'\zs')) let result=(temp>result ? temp : result) endfor return result else return len(b:csv_fixed_width_cols) endif endfu fu! ColWidth(colnr) "{{{3 " Return the width of a column " Internal function, not called from extern, " does not work with fixed width columns let width=20 "Fallback (wild guess) let tlist=[] if !exists("b:csv_fixed_width_cols") if !exists("b:csv_list") let b:csv_list=getline(1,'$') call map(b:csv_list, 'split(v:val, b:col.''\zs'')') endif for item in b:csv_list call add(tlist, item[a:colnr-1]) endfor try " we have a list of the first 10 rows " Now transform it to a list of field a:colnr " and then return the maximum strlen " That could be done in 1 line, but that would look ugly "call map(list, 'split(v:val, b:col."\\zs")[a:colnr-1]') call map(tlist, 'substitute(v:val, ".", "x", "g")') call map(tlist, 'strlen(v:val)') return max(tlist) catch return width endtry else if a:colnr > 0 return b:csv_fixed_width_cols[a:colnr] - b:csv_fixed_width_cols[(a:colnr - 1)] endif endif endfu fu! ArrangeCol(first, last, bang) range "{{{3 "TODO: Why doesn't that work? " is this because of the range flag? " It's because of the way, Vim works with " a:firstline and a:lastline parameter, therefore " explicitly give the range as argument to the function if exists("b:csv_fixed_width_cols") " Nothing to do call Warn("ArrangeColumn does not work with fixed width column!") return endif let cur=winsaveview() if a:bang || !exists("b:col_width") " Force recalculation of Column width call CalculateColumnWidth() endif if &ro " Just in case, to prevent the Warning " Warning: W10: Changing read-only file setl noro endif exe a:first . ',' . a:last .'s/' . (b:col) . \ '/\=Columnize(submatch(0))/' . (&gd ? '' : 'g') " Clean up variables, that were only needed for Columnize() function unlet! s:columnize_count s:max_cols setl ro call winrestview(cur) endfu fu! CalculateColumnWidth() "{{{3 " Internal function, not called from external, " does not work with fixed width columns let b:col_width=[] " Force recalculating the Column width unlet! b:csv_list let s:max_cols=MaxColumns() for i in range(1,s:max_cols) call add(b:col_width, ColWidth(i)) endfor " delete buffer content in variable b:csv_list, " this was only necessary for calculating the max width unlet! b:csv_list endfu fu! Columnize(field) "{{{3 " Internal function, not called from external, " does not work with fixed width columns if !exists("s:columnize_count") let s:columnize_count = 0 endif if !exists("s:max_cols") let s:max_cols = len(b:col_width) endif " convert zero based indexed list to 1 based indexed list, " Default: 20 width, in case that column width isn't defined " Careful: Keep this fast! Using "let width=get(b:col_width,WColumn()-1,20) " is too slow, so we are using: let width=get(b:col_width, (s:columnize_count % s:max_cols), 20) let s:columnize_count += 1 if !exists("g:csv_no_multibyte") && \ match(a:field, '[^ -~]') != -1 " match characters outside the ascii range let a = split(a:field, '\zs') let add = eval(join(map(a, 'len(v:val)'), '+')) let add -= len(a) else let add = 0 endif if width + add + 1 == strlen(a:field) " Column has correct length, don't use printf() return a:field endif " Add one for the frame " plus additional width for multibyte chars, " since printf(%*s..) uses byte width! let width = width + add + 1 return printf("%*s", width , a:field) endfun fu! GetColPat(colnr, zs_flag) "{{{3 " Return Pattern for given column if a:colnr > 1 if !exists("b:csv_fixed_width_cols") let pat=b:col . '\{' . (a:colnr) . '\}' else if a:colnr >= len(b:csv_fixed_width_cols) " Get last column let pat='\%' . b:csv_fixed_width_cols[-1] . 'c.*' else let pat='\%' . b:csv_fixed_width_cols[(a:colnr - 1)] . \ 'c.\{-}\%' . b:csv_fixed_width_cols[a:colnr] . 'c' endif endif else if !exists("b:csv_fixed_width_cols") let pat=b:col else let pat='\%' . b:csv_fixed_width_cols[0] . 'c.\{-}\%' . \ b:csv_fixed_width_cols[1] . 'c' endif endif return pat . (a:zs_flag ? '\zs' : '') endfu fu! SplitHeaderLine(lines, bang, hor) "{{{3 if exists("b:csv_fixed_width_cols") call Warn("Header does not work with fixed width column!") return endif if !a:bang " A Split Header Window already exists, " first close the already existing Window if exists("b:CSV_SplitWindow") call SplitHeaderLine(a:lines, 1, a:hor) endif " Split Window let _stl = &l:stl let _sbo = &sbo if a:hor setl scrollopt=hor scrollbind let lines = empty(a:lines) ? 1 : a:lines abo sp 1 exe "resize" . lines setl scrollopt=hor scrollbind winfixheight "let &l:stl=repeat(' ', winwidth(0)) let &l:stl="%#Normal#".repeat(' ',winwidth(0)) " Highlight first row let win = winnr() else setl scrollopt=ver scrollbind 0 let b=b:col let a=[] let a=CopyCol('',1) " Force recalculating columns width unlet! b:csv_list let width = ColWidth(1) let b=b:col abo vsp +enew let b:col=b call append(0, a) $d _ sil %s/.*/\=printf("%.*s", width, submatch(0))/eg 0 exe "vert res" width setl scrollopt=ver scrollbind winfixwidth setl buftype=nowrite bufhidden=hide noswapfile nobuflisted let win = winnr() endif call matchadd("CSVHeaderLine", b:col) exe "wincmd p" let b:CSV_SplitWindow = win else " Close split window if !exists("b:CSV_SplitWindow") return endif exe b:CSV_SplitWindow . "wincmd w" if exists("_stl") let &l_stl = _stl endif if exists("_sbo") let &sbo = _sbo endif setl noscrollbind wincmd c unlet! b:CSV_SplitWindow endif endfu fu! SplitHeaderToggle(hor) "{{{3 if !exists("b:CSV_SplitWindow") :call SplitHeaderLine(1,0,a:hor) else :call SplitHeaderLine(1,1,a:hor) endif endfu " TODO: from here on add logic for fixed-width csv files! fu! MoveCol(forward, line) "{{{3 " Move cursor position upwards/downwards left/right let colnr=WColumn() let maxcol=MaxColumns() let cpos=getpos('.')[2] if !exists("b:csv_fixed_width_cols") call search(b:col, 'bc', line('.')) endif let spos=getpos('.')[2] " Check for valid column " a:forward == 1 : search next col " a:forward == -1: search prev col " a:forward == 0 : stay in col if colnr - v:count1 >= 1 && a:forward == -1 let colnr -= v:count1 elseif colnr - v:count1 < 1 && a:forward == -1 let colnr = 0 elseif colnr + v:count1 <= maxcol && a:forward == 1 let colnr += v:count1 elseif colnr + v:count1 > maxcol && a:forward == 1 let colnr = maxcol + 1 endif let line=a:line if line < 1 let line=1 elseif line > line('$') let line=line('$') endif " Generate search pattern if colnr == 1 let pat = '^' . GetColPat(colnr-1,0) "let pat = pat . '\%' . line . 'l' elseif (colnr == 0) || (colnr == maxcol + 1) if !exists("b:csv_fixed_width_cols") let pat=b:col else if a:forward > 0 " Move forwards let pat=GetColPat(1, 0) else " Move backwards let pat=GetColPat(maxcol, 0) endif endif else if !exists("b:csv_fixed_width_cols") let pat='^'. GetColPat(colnr-1,1) . b:col else let pat=GetColPat(colnr,0) endif "let pat = pat . '\%' . line . 'l' endif " Search " move left/right if a:forward > 0 call search(pat, 'W') elseif a:forward < 0 call search(pat, 'bWe') " Moving upwards/downwards elseif line >= line('.') call search(pat . '\%' . line . 'l', '', line) " Move to the correct screen column " This is a best effort approach, we might still " leave the column (if the next column is shorter) if !exists("b:csv_fixed_width_cols") let a = getpos('.') let a[2]+= cpos-spos else let a = getpos('.') let a[2] = cpos endif call setpos('.', a) elseif line < line('.') call search(pat . '\%' . line . 'l', 'b', line) " Move to the correct screen column if !exists("b:csv_fixed_width_cols") let a = getpos('.') let a[2]+= cpos-spos else let a = getpos('.') let a[2] = cpos endif call setpos('.', a) endif endfun fu! SortComplete(A,L,P) "{{{3 return join(range(1,MaxColumns()),"\n") endfun fu! SortList(a1, a2) "{{{3 return a:a1 == a:a2 ? 0 : a:a1 > a:a2 ? 1 : -1 endfu fu! Sort(bang, line1, line2, colnr) range "{{{3 let wsv=winsaveview() if a:colnr =~? 'n' let numeric = 1 else let numeric = 0 endif let col = (empty(a:colnr) || a:colnr !~? '\d\+') ? WColumn() : a:colnr+0 if col != 1 if !exists("b:csv_fixed_width_cols") let pat= '^' . GetColPat(col-1,1) . b:col else let pat= '^' . GetColPat(col,0) endif else let pat= '^' . GetColPat(col,0) endif exe a:line1 ',' a:line2 . "sort" . (a:bang ? '!' : '') . \' r ' . (numeric ? 'n' : '') . ' /' . pat . '/' call winrestview(wsv) endfun fu! CSV_WCol() "{{{3 return printf(" %d/%d", WColumn(), MaxColumns()) endfun fu! CopyCol(reg, col) "{{{3 " Return Specified Column into register reg let col = a:col == "0" ? WColumn() : a:col+0 let mcol = MaxColumns() if col == '$' || col > mcol let col = mcol endif let a=getline(1, '$') if !exists("b:csv_fixed_width_cols") call map(a, 'split(v:val, ''^'' . b:col . ''\zs'')[col-1]') else call map(a, 'matchstr(v:val, GetColPat(col, 0))') endif if a:reg =~ '[-"0-9a-zA-Z*+]' "exe ':let @' . a:reg . ' = "' . join(a, "\n") . '"' " set the register to blockwise mode call setreg(a:reg, join(a, "\n"), 'b') else return a endif endfu fu! MoveColumn(start, stop, ...) range "{{{3 " Move column behind dest " Explicitly give the range as argument, " cause otherwise, Vim would move the cursor let wsv = winsaveview() let col = WColumn() let max = MaxColumns() " If no argument is given, move current column after last column let source=(exists("a:1") && a:1 > 0 && a:1 <= max ? a:1 : col) let dest =(exists("a:2") && a:2 > 0 && a:2 <= max ? a:2 : max) " translate 1 based columns into zero based list index let source -= 1 let dest -= 1 if source >= dest call Warn("Destination column before source column, aborting!") return endif " Swap line by line, instead of reading the whole range into memory for i in range(a:start, a:stop) if !exists("b:csv_fixed_width_cols") let fields=split(getline(i), b:col . '\zs') else let fields=[] for j in range(1, max, 1) call add(fields, matchstr(getline(i), GetColPat(j,0))) endfor endif " Add delimiter to destination column, in case there was none, " remove delimiter from source, in case destination did not have one if matchstr(fields[dest], '.$') !~? b:delimiter let fields[dest] = fields[dest] . b:delimiter if matchstr(fields[source], '.$') =~? b:delimiter let fields[source] = substitute(fields[source], \ '^\(.*\).$', '\1', '') endif endif let fields= (source == 0 ? [] : fields[0 : (source-1)]) \ + fields[ (source+1) : dest ] \ + [ fields[source] ] + fields[(dest+1):] call setline(i, join(fields, '')) endfor call winrestview(wsv) endfu fu! SumColumn(list) "{{{3 return eval(join(a:list, '+')) endfu fu! csv#EvalColumn(nr, func, first, last) range "{{{3 let save = winsaveview() let col = (empty(a:nr) ? WColumn() : a:nr) let start = a:first - 1 let stop = a:last - 1 let column = CopyCol('', col)[start : stop] " Delete delimiter call map(column, 'substitute(v:val, b:delimiter, "", "g")') try let result=call(function(a:func), [column]) return result catch " Evaluation of expression failed echohl Title echomsg "Evaluating" matchstr(a:func, '[a-zA-Z]\+$') \ "failed for column" col . "!" echohl Normal return '' finally call winrestview(save) endtry endfu fu! DoForEachColumn(start, stop, bang) range "{{{3 " Do something for each column, " e.g. generate SQL-Statements, convert to HTML, " something like this " TODO: Define the function " needs a csv_pre_convert variable " csv_post_convert variable " csv_convert variable " result contains converted buffer content let result = [] if !exists("g:csv_convert") call Warn("You need to define how to convert your data using" . \ "the g:csv_convert variable, see :h csv-convert") return endif if exists("g:csv_pre_convert") && !empty(g:csv_pre_convert) call add(result, g:csv_pre_convert) endif for item in range(a:start, a:stop, 1) let t = g:csv_convert let line = getline(item) let context = split(g:csv_convert, '%s') let columns = len(context) if columns > MaxColumns() let columns = MaxColumns() elseif columns == 1 call Warn("No Columns defined in your g:csv_convert variable, Aborting") return endif if !exists("b:csv_fixed_width_cols") let fields=split(line, b:col . '\zs') if a:bang call map(fields, 'substitute(v:val, b:delimiter . \ ''\?$'' , "", "")') endif else let fields=[] for j in range(1, columns, 1) call add(fields, matchstr(line, GetColPat(j,0))) endfor endif for j in range(1, columns, 1) let t=substitute(t, '%s', fields[j-1], '') endfor call add(result, t) endfor if exists("g:csv_post_convert") && !empty(g:csv_post_convert) call add(result, g:csv_post_convert) endif new call append('$', result) 1d _ endfun fu! PrepareDoForEachColumn(start, stop, bang) range"{{{3 let pre = exists("g:csv_pre_convert") ? g:csv_pre_convert : '' let g:csv_pre_convert=input('Pre convert text: ', pre) let post = exists("g:csv_post_convert") ? g:csv_post_convert : '' let g:csv_post_convert=input('Post convert text: ', post) let convert = exists("g:csv_convert") ? g:csv_convert : '' let g:csv_convert=input("Converted text, use %s for column input:\n", convert) call DoForEachColumn(a:start, a:stop, a:bang) endfun fu! CommandDefinitions() "{{{3 if !exists(":WhatColumn") "{{{4 command! -buffer -bang WhatColumn :echo WColumn(0) endif if !exists(":NrColumns") "{{{4 command! -buffer NrColumns :echo MaxColumns() endif if !exists(":HiColumn") "{{{4 command! -buffer -bang -nargs=? HiColumn :call HiCol(.) endif if !exists(":SearchInColumn") "{{{4 command! -buffer -nargs=* SearchInColumn :call SearchColumn() endif if !exists(":DeleteColumn") "{{{4 command! -buffer -nargs=? -complete=custom, \SortComplete DeleteColumn :call DelColumn() endif if !exists(":ArrangeColumn") "{{{4 command! -buffer -range -bang ArrangeColumn \ :call ArrangeCol(,, 0) endif if !exists(":InitCSV") "{{{4 command! -buffer InitCSV :call Init() endif if !exists(":Header") "{{{4 command! -buffer -bang -nargs=? Header :call SplitHeaderLine(,0,1) endif if !exists(":VHeader") "{{{4 command! -buffer -bang -nargs=? VHeader :call SplitHeaderLine(,0,0) endif if !exists(":HeaderToggle") "{{{4 command! -buffer HeaderToggle :call SplitHeaderToggle(1) endif if !exists(":VHeaderToggle") "{{{4 command! -buffer VHeaderToggle :call SplitHeaderToggle(0) endif if !exists(":Sort") "{{{4 command! -buffer -nargs=* -bang -range=% -complete=custom, \SortComplete Sort :call \Sort(0,,,) endif if !exists(":Column") "{{{4 command! -buffer -count -register Column :call CopyCol( \ empty() ? '"' : ,) endif if !exists(":MoveColumn") "{{{4 command! -buffer -range=% -nargs=* -complete=custom,SortComplete \ MoveColumn :call MoveColumn(,,) endif if !exists(":SumCol") "{{{4 command! -buffer -nargs=? -range=% -complete=custom,SortComplete \ SumCol :echo csv#EvalColumn(, "SumColumn", ,) endif if !exists(":ConvertData") "{{{4 command! -buffer -bang -nargs=? -range=% \ -complete=custom,SortComplete ConvertData \ :call PrepareDoForEachColumn(,, 0) endif endfu fu! CSVMappings() "{{{3 noremap W :call MoveCol(1, line('.')) noremap E :call MoveCol(-1, line('.')) noremap K :call MoveCol(0, line('.')-v:count1) noremap J :call MoveCol(0, line('.')+v:count1) " Remap J and K to a sane backup noremap J J noremap K K " Map C-Right and C-Left as alternative to W and E map W map E map H E map L W map K map J endfu " end function definition "}}}2 " Initialize Plugin "{{{2 call Init() let &cpo = s:cpo_save unlet s:cpo_save " Vim Modeline " {{{2 " vim: set foldmethod=marker: doc/ft-csv.txt [[[1 958 *ft-csv.txt* For Vim version 7.3 Last change: 2011 Feb 17 Author: Christian Brabandt Version: 0.18 Homepage: http://www.vim.org/scripts/script.php?script_id=2830 The VIM LICENSE applies to the CSV filetype plugin (see |copyright|). NO WARRANTY, EXPRESS OR IMPLIED. USE AT-YOUR-OWN-RISK. *csv-toc* 1. Introduction.................................|csv-intro| 2. Installation.................................|csv-installation| 3. CSV Commands.................................|csv-commands| 3.1 WhatColumn..............................|WhatColumn_CSV| 3.2 NrColumns...............................|NrColumns_CSV| 3.3 SearchInColumn..........................|SearchInColumn_CSV| 3.4 HiColumn................................|HiColumn_CSV| 3.5 ArrangeColumn...........................|ArrangeColumn_CSV| 3.6 DeleteColumn............................|DeleteColumn_CSV| 3.7 InitCSV.................................|InitCSV| 3.8 Header..................................|Header_CSV| 3.9 Sort....................................|Sort_CSV| 3.10 CopyColumn.............................|Copy_CSV| 3.11 MoveColumn.............................|MoveCol_CSV| 3.12 Sum of a column........................|SumCol_CSV| 3.13 Normal mode commands...................|csv-motion| 3.14 Convert CSV file.......................|csv-convert| 4. CSV Filetype configuration...................|csv-configuration| 4.1 Delimiter...............................|csv-delimiter| 4.2 Column..................................|csv-column| 4.3 HiGroup.................................|csv-higroup| 4.4 Strict Columns..........................|csv-strict| 4.5 Multibyte Chars.........................|csv-mbyte| 4.6 Concealing..............................|csv-conceal| 4.7 Newlines................................|csv-newline| 4.8 Highlight column automatically..........|csv-hicol| 4.9 Fixed width columns.....................|csv-fixedwidth| 5. CSV Tips and Tricks..........................|csv-tips| 5.1 Statusline..............................|csv-stl| 5.2 Slow ArrangeCol.........................|csv-slow| 5.3 Defining custom aggregate functions.....|csv-aggregate-functions| 6. CSV Changelog................................|csv-changelog| ============================================================================== 1. Introduction *csv-intro* This plugin is used for handling column separated data with Vim. Usually those files are called csv files and use the ',' as delimiter, though sometimes they use e.g. the '|' or ';' as delimiter and there also exists fixedwidth columns. The aim of this plugin is to ease handling these kinds of files. This is a filetype plugin for CSV files. It was heavily influenced by the Vim Wiki Tip667 (http://vim.wikia.com/wiki/VimTip667), though it works differently. For instructions on installing this file, type :help add-local-help |add-local-help| inside Vim. For a screenshot, of how the plugin can be used, see http://www.256bit.org/~chrisbra/csv.gif ============================================================================== 2. Installation *csv-installation* In order to have vim automatically detect csv files, you need to have |ftplugins| enabled (e.g. by having this line in your |.vimrc| file: > :filetype plugin on < The plugin already sets up some logic, to detect CSV files. By default, the plugin recognizes *.csv and *.dat files as csv filetype. In order that the csv filetype plugin is loaded correctly, vim needs to be enabled to load |filetype-plugins|. This can be ensured, by putting a line like this into your |.vimrc|: > :filetype plugin on < (see also |filetype-plugin-on|). In case this did not work, you need to setup vim like this: To have Vim automatically detect csv files, you need to do the following. 1) Create your user runtime directory if you do not have one yet. This directory needs to be in your 'runtime' path. In Unix this would typically the ~/.vim directory, while in Windows this is usually your ~/vimfiles directory. Use :echo expand("~") to find out, what Vim thinks your user directory is. To create this directory, you can do: > :!mkdir ~/.vim < for Unix and > :!mkdir ~/vimfiles < for Windows. 2) In that directory you create a file that will detect csv files. > if exists("did_load_csvfiletype") finish endif let did_load_csvfiletype=1 augroup filetypedetect au! BufRead,BufNewFile *.csv,*.dat setfiletype csv augroup END < You save this file as "filetype.vim" in your user runtime diretory: > :w ~/.vim/filetype.vim < 3) To be able to use your new filetype.vim detection, you need to restart Vim. Vim will then load the csv filetype plugin for all files whose names end with .csv. ============================================================================== 3. Commands *csv-commands* The CSV ftplugin provides several Commands: 3.1 WhatColumn *WhatColumn_CSV* -------------- If you would like to know, on which column the cursor is, use > :WhatColumn Use the bang attribute, if you have a heading in the first line and you want to know the name of the column in which the cursor is: > :WhatColumn! < 3.2 NrColumns *NrColumns_CSV* -------------- :NrColumns outputs the maximum number of columns available. It does this by testing the first 10 lines for the number of columns. This usually should be enough. 3.3 SearchInColumn *SearchInColumn_CSV* ------------------ Use :SearchInColumn to search for a pattern within a specific column. The usage is: > :SearchInColumn [] /{pat}/ < So if you would like to search in Column 1 for the word foobar, you enter > :SearchInColumn 1 /foobar/ Instead of / as delimiter, you can use any other delimiter you like. If you don't enter a column, the current column will be used. 3.4 HiColumn *HiColumn_CSV* ------------ :HiColumn can be used to highlight Column . Currently the plugin uses the WildMenu Highlight Group. If you would like to change this, you need to define the variable |g:csv_hiGroup|. If you do not specify a , HiColumn will highlight the column on which the cursor is. Use > :HiColumn! to remove any highlighting. If you want to automatically highlight a column, see |csv-hicol| *:ArrangeColumn* 3.5 ArrangeColumn *ArrangeColumn_CSV* ----------------- If you would like all columns to be visually arranged, you can use the > :ArrangeColumn[!] command. Beware, that this will change your file and depending on the size of your file may slow down Vim significantly. This is highly experimental. :ArrangeCommand will try to vertically align all columns by their maximum column size. Use the bang attribute to force recalculating the column width. This is slower, but especially if you have modified the file, this will correctly calculate the width of each column so that they can be correctly aligned. If no column width has been calculated before, the width will be calculated, even if the '!' has not been given. Note, this can be very slow on large files or many columns (see |csv-slow| on how to increase performance for this command). To prevent you from accidently changing your csv file, the buffer will be set 'readonly' afterwards. Note: this command does not work for fixed width columns |csv-fixedwidth| 3.6 DeleteColumn *DeleteColumn_CSV* ---------------- The command :DeleteColumn can be used to delete a specific column. > :DeleteColumn 2 will delete column 2. If you don't specify a column number, it will delete the column on which the cursor is. 3.7 InitCSV *InitCSV* ----------- Reinitialize the Plugin. Use this, if you have changed the configuration of the plugin (see |csv-configuration| ). 3.8 Header lines *Header_CSV* ---------------- This command splits the csv-buffer and adds a window, that holds a small fraction of the csv file. This is useful, if the first line contains some kind of a heading and you want always to display it. This works similar to fixing a certain line at the top. As optional argument, you can give the number of columns from the top, that shall be displayed. By default, 1 is used. Use the '!' to close this window. So this > :Header 3 opens at the top a split window, that holds the first 3 lines, is fixed and horizontally 'scrollbind'ed to the csv window and highlighted using the CSVHeaderLine highlighting. To close the header window, use > :Header! Note, this won't work with linebreaks in the column. Note also, that if you already have a horizontal header window (|VHeader_CSV|), this command will close the horizontal Header window. This is because of a limitation of Vim itsself, which doesn't allow to sync the scrolling between two windows horizontally and at the same time have another window only sync its scrolling vertically. Note: this command does not work for fixed width columns |csv-fixedwidth| *VHeader_CSV* If you want a vertical header line, use :VHeader. This works similar to the |Header_CSV| command, except that it will open a vertical split window with the first column always visible. It will always open the first column in the new split window. Use the '!' to close the window. Note, this won't work with linebreaks in the column. Note also: this command does not work for fixed width columns |csv-fixedwidth| *VHeaderToggle_CSV* *HeaderToggle_CSV* Use the :HeaderToggle and :VHeaderToggle command to toggle displaying the horizontal or vertical header line. 3.9 Sort *Sort_CSV* -------- This command can be used to sort the csv file on a certain column. If no range is given, is sorts the whole file. Specify the column number to sort on as argument. Use the '!' attribute to reverse the sort order. For example, the following command sorts line 1 til 10 on the 3 column > :1,10Sort 3 While this command > :1,10Sort! 3 reverses the order based on column 3. Instead of a column, you can give the flag 'n' to have it sort numerically. When no column number is given, it will sort by the column, on which the cursor is currently. 3.10 Copy Column *Copy_CSV* ---------------- If you need to copy a specific column, you can use the command :Column > :[N]Column [a] Copy column N into register a. If you don't specify N, the column of the current cursor position is used. If no register is given, the default register |quotequote| is used. 3.11 Move A Column *MoveCol_CSV* ------------------ You can move one column behind another column by using the :MoveColumn command > :[range]MoveColumn [source] [dest] This moves the column number source behind column nr destination. If both arguments are not given, move the column on which the cursor is behind the last column. If [range] is not given, MoveColumn moves all columns, otherwise, it moves the columns only for the lines within the range, e.g. given that your first line is a header line, which you don't want to change > :2,$MoveColumn 1 $ this would move column 1 behind the last column, while keeping the header line as is. 3.12 Sum of a Column *SumCol_CSV* -------------------- You can let Vim output the sum of a column using the :SumCol command > :[range]SumCol [nr] This outputs the result of the column within the range given. If no range is given, this will calculate the sum of the whole column. If is not given, this calculate the sum for the column the cursor is on. See also |csv-aggregate-functions| 3.13 Normal mode commands *csv-motion* ------------------------- The csv filetype plugin redefines the following keys as: or L or W Move [count] field forwards or E or H Move [count] field backwards or K Move [count] lines upwards within the same column or J Move [count] lines downwards within the same column Note, that the K and J overlap Vim's default mapping for |J| and |K| respectively. Therefore, this functionality has been mapped to a sane default of J and K. If you haven't changed the || or || variables, those the is equival to a single backslash '\', e.g. \K would run the lookup function on the word under the cursor and \J would join this line with the previous line. *ConvertData_CSV* 3.14 Converting a CSV File *csv-convert* -------------------------- You can convert your CSV file to a different format with the command > ConvertData Use the the ! attribute, to convert your data without the delimiter. This command will interactively ask you for the definition of 3 variables. After which it will convert your csv file into a new format, defined by those 3 variables and open the newly created file in a new window. Those 3 variables define how the text converted. First, You need to define what has to be done, before converting your column data. That is done with the "pre convert" variable. The content of this variable will be put in front of the new document. Second, you define, what has to be put after the converted content of your column data. This happens with the "post convert" variable. Basically the contents of this variable will be put after processing the columns. Last, the columns need to be converted into your format. For this you can specify a printf() format like string, that defines how your data will be converted. You can use '%s' to specify placeholders, which will later be replaced by the content of the actual column. For example, suppose you want to convert your data into HTML, then you first call the > ConvertData At this point, Vim will ask you for input. First, you need to specify, what needs to be done before processing the data: Pre convert text: ` This would specify to put the HTML Header before the actual data can be processed. If the variable g:csv_pre_convert is already defined, Vim will already show you its' content as default value. Simply pressing Enter will use this data. After that, Vim asks, what the end of the converted file needs to look like: Post convert text:
So here you are defining how to finish up the HTML file. If the variable g:csv_post_convert is already defined, Vim will already show you its' content as default value which you can confirm by pressing Enter. Last, you define, how your columns need to be converted. Again, Vim asks you for how to do that: Converted text, use %s for column input: %s%s%s This time, you can use '%s' expandos. They tell Vim, that they need to be replaced by the actual content of your file. It does by going from the first column in your file and replacing it with the corresponding %s in that order. If there are less '%s' expandos then columns in your file, Vim will skip the columns, that are not used. Again If the variable g:csv__convert is already defined, Vim will already show you its' content as default value which you can confirm by pressing Enter. After you hit Enter, Vim will convert your data and put it into a new window. It may look like this: ` ` `
1,2,3,
2,2,4,
` Note, this is only a proof of concept. A better version of converting your data to HTML is bundled with Vim (|:TOhtml|). But may be you want your data converted into SQL-insert statements. That could be done like this: > ConvertData! Pre convert text: ` (Leave this empty. It won't be used). Post convert text: Commit;` After inserting the data, commit it into the database. Converted text, use %s for column input: ` Insert into table foobar values ('%s', '%s', %s); ` Note, that the last argument is not included within single quotation marks, since in this case the data is assumed to be integer and won't need to be quoted for the database. After hitting Enter, a new Window will be opened, which might look like this: Insert into table foobar values('Foobar', '2', 2011); Insert into table foobar values('Bar', '1', 2011); ... Commit; Since the command was used with the bang attribute (!), the converted data doesn't include the column delimiters. Now you can copy it into your database, or further manipulate it. ============================================================================== 4. CSV Configuration *csv-configuration* The CSV plugin tries to automatically detect the field delimiter for your file, cause although often the file is called CSV (comma separated values), a semicolon is actually used. The column separator is stored in the buffer-local variable b:delimiter. This delimiter is heavily used, because you need it to define a column. Almost all commands use this variable therefore. 4.1 Delimiter *csv-delimiter* ------------- To override the automatic detection of the plugin and define the separator manually, use: > :let g:csv_delim=',' to let the comma be the delimiter. This sets the buffer local delimiter variable b:delimiter. If your file does not consist of delimited columns, but rather is a fixed width csv file, see |csv-fixedwidth| for configuring the plugin appropriately. If you changed the delimiter, you should reinitiliaze the plugin using |InitCSV| 4.2 Column *csv-column* ---------- The definition, of what a column is, is defined as buffer-local variable b:col. By default this variable is initialized to: > let b:col='\%(\%([^' . b:delimiter . ']*"[^"]*"[^' . b:delimiter . ']*' \. b:delimiter . '\)\|\%([^' . b:delimiter . ']*\%(' . b:delimiter \. '\|$\)\)\)' This should take care of quoted delimiters within a column. Those should obviously not count as a delimiter. This regular expression is quite complex and might not always work on some complex cases (e.g. linebreaks within a field, see RFC4180 for some ugly cases that will probably not work with this plugin). If you changed the b:delimiter variable, you need to redefine the b:col variable, cause otherwise it will not reflect the change. To change the variable from the comma to a semicolon, you could call in your CSV-Buffer this command: > :let b:col=substitute(b:col, ',', ';', 'g') Check with :echo b:col, if the definition is correct afterwards. You can also force the plugin to use your own defined regular expression as column. That regular expression should include the delimiter for the columns. To define your own regular expression, set the g:csv_col variable: > let g:csv_col='[^,]*,' This defines a column as a field delimited by the comma (where no comma can be contained inside a field), similar to how |csv-strict| works. You should reinitialize the plugin afterwards |InitCSV| 4.3 Highlighting Group *csv-higroup* ---------------------- By default the csv ftplugin uses the WildMenu highlighting Group to define how the |HiColumn| command highlights columns. If you would like to define a different highlighting group, you need to set this via the g:csv_hiGroup variable. You can e.g. define it in your |.vimrc|: > :let g:csv_hiGroup = "IncSearch" You need to restart Vim, if you have changed this variable or use |InitCSV| The |hl-Title| highlighting is used for the Header line that is created by the |Header_CSV| command. If you prefer a different highlighting, set the g:csv_hiHeader variable to the prefered highlighting: > let g:csv_hiHeader = 'Pmenu' < This would set the header window to the |hl-Pmenu| highlighting, that is used for the popup menu. To disable the custom highlighting, simply |unlet| the variable: > unlet g:csv_hiHeader You should reinitialize the plugin afterwards |InitCSV| 4.4 Strict Columns *csv-strict* ------------------ The default regular expression to define a column is quite complex |csv-column|. This slows down the processing and makes Vim use more memory and it could still not fit to your specific use case. If you know, that in your data file, the delimiter cannot be contained inside the fields quoted or escaped, you can speed up processing (this is quite noticeable when using the |ArrangeColumn_CSV| command) by setting the g:csv_strict_columns variable: > let g:csv_strict_columns = 1 This would define a column as this regex: > let b:col = '\%([^' . b:delimiter . ']*' . b:delimiter . '\|$\)' Much simpler then the default column definition, isn't it? See also |csv-column| and |csv-delimiter| You can disable the effect if you |unlet| the variable: > unlet g:csv_strict_columns You should reinitialize the plugin afterwards |InitCSV| 4.5 Multibyte Chars *csv-mbyte* ------------------- Unfortunately, when using the |ArrangeColumn_CSV| command, multibyte chars make some trouble, because internally Vim uses bytes to specify the width of a column. The CSV plugin tries to workaround that, by calculating the byte width of each column, before aligning them. This is quite expensive and can slow down processing. If you know, your data file only contains pure ASCII chars (or you simply don't care, if some lines a off a little bit), set the g:csv_no_multibyte variable: > let g:csv_no_multibyte = 1 And to force calculating the byte width of each column |unlet| the variable: > unlet g:csv_no_multibyte You should reinitialize the plugin afterwards |InitCSV| 4.6 Concealing *csv-syntax* *csv-conceal* ------------------- The CSV plugin comes with a function to syntax highlight csv files. Basically allt it does is highlight the columns and the header line. By default, the delimiter will not be displayed, if Vim supports |conceal| of syntax items and instead draws a vertical line. If you don't want that, simply set the g:csv_noconceal variable in your .vimrc > let g:csv_no_conceal = 1 and to disable it, simply unlet the variable > unlet g:csv_no_conceal You should reinitialize the plugin afterwards |InitCSV| If you want to customize the syntax colors, you can define your own groups. The CSV plugin will use already defined highlighting groups, if they are already defined, otherwise it will define its own defaults which should be visible with 8, 16, 88 and 256 color terminals. For that it uses the CSVColumnHeaderOdd and CSVColumnHeaderEven highlight groups for syntax coloring the first line. All other lines get either the CSVColumnOdd or CSVColumnEven highlighting. In case you want to define your own highlighting groups, you can define your own syntax highlighting like this in your |.vimrc| > hi CSVColumnEven term=bold ctermbg=4 guibg=DarkBlue hi CSVColumnOdd term=bold ctermbg=5 guibg=DarkMagenta hi CSVColumnHeaderEven ... hi CSVColumnHeaderOdd ... < Note, these changes won't take effect, until you restart Vim. 4.7 Newlines *csv-newline* ------------ RFC4180 allows newlines in double quoted strings. By default, the csv-plugin won't recognize newlines inside fields. It is however possible to make the plugin aware of newlines within quoted strings. To enbale this, set > let g:csv_nl = 1 and to disable it again, simply unset the variable > unlet g:csv_nl It is a good idea to reinitialize the plugin afterwards |InitCSV| Note, this might not work correctly in all cases. The syntax highlighting seems to change on cursor movements. This could possibly be a bug in the syntax highlighting engine of Vim. Also, |WhatColumn_CSV| can't handle newlines inside fields and will most certainly be wrong. 4.8 Highlight column automatically *csv-hicol* ---------------------------------- You can let vim automatically highlight the column on which the cursor is. This works by defining an |CursorMoved| autocommand to always highlight the column, when the cursor is moved in normal mode. Note, this does not update the highlighting, if the Cursor is moved in Insert mode. To enable this, define the g:csv_highlight_column variable like this > let g:csv_highlight_column = 'y' and to disable it again, simply unset the variable > unlet g:csv_highlight_column It is a good idea to reinitialize the plugin afterwards |InitCSV| 4.9 Fixed width columns *csv-fixedwidth* ----------------------- Sometimes there are no real columns, but rather the file is fixed width with no distinct delimiters between each column. The CSV plugin allows you to handle such virtual columns like csv columns, if you define where each column starts. You can do this, by setting the buffer-local variable b:csv_fixed_width like this > let b:csv_fixed_width="1,5,9,13,17,21" This defines that each column starts at multiples of 4. Be sure, to issue this command in the buffer, that contains your file, otherwise, it won't have an effect, since this is a buffer-local option (|local-option|) Except for |ArrangeColumn_CSV| and the |Header_CSV| commands, all commands work in either mode. Those two commands won't do anything in the case of fixedwidth columns, since they don't really make sense here. After setting this variable, you should reinitialize the plugins using |InitCSV| ============================================================================== 5. CSV Tips and Tricks *csv-tips* Here, there you'll find some small tips and tricks that might help when working with CSV files. 5.1 Statusline *csv-stl* -------------- Suppose you want to include the column, on which the cursor is, into your statusline. You can do this, by defining in your .vimrc the 'statusline' like this: > function MySTL() if has("statusline") hi User1 term=standout ctermfg=0 ctermbg=11 guifg=Black guibg=Yellow let stl = ... if exists("*CSV_WCol") let csv = '%1*%{&ft=~"csv" ? CSV_WCol() : ""}%*' else let csv = '' endif return stl.csv endif endfunc set stl=%!MySTL() < This will draw in your statusline right aligned the current column and max column (like 1/10), if you are inside a CSV file. The column info will be drawn using the User1 highlighting (|hl-User1|), that has been defined in the second line of the function. In the third line of your function, put your desired 'statusline' settings as |expression|. Note the section starting with 'if exists(..)' guards against not having loaded the filetype plugin. 5.2 Slow ArrangeCol *csv-slow* ------------------- Processing a csv file using |ArrangeColumn_CSV| can be quite slow, because Vim needs to calculate the width for each column and then replace each column by itself widened by spaces to the optimal length. Unfortunately, csv files tend to be quite big. Remember, for a file with 10,000 lines and 50 columns Vim needs to process each cell, which accumulates to 500,000 substitutions. It might take some time, until Vim is finished. You can speed up things a little bit, if you omit the '!' attribute to the |ArrangeColumn| (but this will only work, if the width has been calculated before, e.g. by issuing a :1ArrangeColumn command to arrange only the first line. Additionally you can also configure how this command behaves by setting some configuration variables. Here are some performance meassurements on how the various configuration settings influence the |ArrangeColumn_CSV| command on a file with 34 columns (on a 2.2GHz Core2Duo processor): Lines | default | strict¹ | multibyte² | strict + multibyte³ ------------------------------------------------------------------------ 1000 | 6.93 s | 5.53 s | 6.76 s | 5.66 s 5000 | 15.2 s | 8.52 s | 14.27 s | 8.56 s 10000 | 36.2 s | 24.67 s | 36.11 s | 24.26 s 50000 | 162,23 s | 93.36 s | 152.25 s | 141.18 s ¹ setting the g:csv_strict_columns variable (|csv-strict|) ² setting the g:csv_no_multibyte variable (|csv-mbyte|) ³ setting the g:csv_no_multibyte variable and g:csv_strict_columns variable Note, this was performed on a quite fast processor. If you need to work with large files, be sure to have enough memory available, cause Vim needs to read in the whole file into memory. You can also try the LargeFile plugin available at http://www.vim.org/scripts/script.php?script_id=1506 which tunes several Vim options (like |syn-off|, 'undolimits', 'fdm' and others). 5.3 Defining custom aggregate functions *csv-aggregate-functions* --------------------------------------- The CSV plugin already defines the |SumCol_CSV|command, to let you calculate the sum of all values of a certain column within a given range. But it may be, that you don't need the sum, but would rather want to have the average of all values within a certain column. You can define your own function and let the plugin call it for a column like this: 1) You define your own custom function in the after directory of your vimruntime path |after-directory| (see also #2 below) > fun! My_CSV_Average(col) let sum=0 for item in a:col let sum+=item endfor return sum/len(a:col) endfun < This function takes a list as argument, and calculates the average for all items in the list. You could also make use of Vim's |eval()| function and write your own Product function like this > fun! My_CSV_Product(col) return eval(join(a:col, '*')) endfun < 2) Now define your own custom command, that calls your custom function for a certain column > command! -buffer -nargs=? -range=% AvgCol \ :echo csv#EvalColumn(, \ "My_CSV_Average", ,) < This command should best be put into a file called csv.vim and put into your ~/.vim/after/ftplugin/ directory. Create directories that don't exist yet. For Windows, this would be the $VIMRUNTIME/vimfiles/after/ftplugin directory. 3) Make sure, your |.vimrc| includes a filetype plugin setting like this > filetype plugin on < This should make sure, that all the necessary scripts are loaded by Vim. After restarting Vim, you can now use your custom command definition :AvgCol. Use a range, for the number of lines you want to evaluate and optionally use an argument to specify which column you want to be evaluated > :2,$AvgCol 7 < This will evaluate the average of column seven (assuming, line 1 is the header line, which should not be taken into account). ============================================================================== 6. CSV Changelog *csv-changelog* 0.18 Aug 30, 2011 {{{1 - fix small typos in documentation - document, that 'K' and 'J' have been remapped and the originial function is available as \K and \J - Delimiters should not be highlighted within a column, only when used as actual delimiters (suggested by Peng Yu, thanks!) - Performance improvements for |:ArrangeColumn| 0.17 Aug 16, 2011 {{{1 - small cosmetic changes - small documentation updates - fold away changelog in help file - Document, that |DeleteColumn_CSV| deletes the column on which the cursor is, if no column number has been specified - Support csv fixed width columns (|csv-fixedwidth|) - Support to interactively convert your csv file to a different format (|csv-convert|) 0.16 Jul 25, 2011 {{{1 - Sort on the range, specified (reported by Peng Yu, thanks!) - |MoveCol_CSV| to move a column behind another column (suggested by Peng Yu, thanks!) - Document how to use custom functions with a column (|csv-aggregate-functions|) - Use g:csv_highlight_column variable, to have Vim automatically highlight the column on which the cursor is (|csv-hicol|) - Header/VHeader command should work better now (|Header_CSV|, |VHeader_CSV|) - Use setreg() for setting the register for the |Column_CSV| command and make sure it is blockwise. - Release 0.14 was not correctly uploaded to vim.org 0.14 Jul 20, 2011 {{{1 - really use g:csv_no_conceal variable (reported by Antonio Ospite, thanks!) - Force redrawing before displaying error messages in syntax script (reported by Antonio Ospite, thanks!) - Make syntax highlighting work better with different terminals (Should work now with 8, 88 and 256 color terminals, tested with linux konsole, xterm and rxvt) (https://github.com/chrisbra/csv.vim/issues/4) - Automatically detect '|' as field separator for csv files 0.13 Mar 14, 2011 {{{1 - documentation update - https://github.com/chrisbra/csv.vim/issues#issue/2 ('splitbelow' breaks |Header_CSV|, fix this; thanks lespea!) - https://github.com/chrisbra/csv.vim/issues#issue/3 ('gdefault' breaks |ArrangeColumn_CSV|, fix this; thanks lespea!) - https://github.com/chrisbra/csv.vim/issues#issue/1 (make syntax highlighting more robust, thanks lespea!) - fix some small annoying bugs - WhatColumn! displays column name 0.12 Feb 24, 2011 {{{1 - bugfix release: - don't use |:noa| when switching between windows - make sure, colwidth() doesn't throw an error 0.11 Feb 24, 2011 {{{1 - new command |Copy_CSV| - |Search_CSV| did not find anything in the last column if no delimiter was given (reported by chroyer) - |VHeader_CSV| display the first column as Header similar to how |Header_CSV| works - |HeaderToggle_CSV| and |VHeaderToggle_CSV| commands that toggle displaying the header lines/columns 0.10 Feb 23, 2011 {{{1 - Only conceal real delimiters - document g:csv_no_conceal variable - document g:csv_nl variable - document conceal feature and syntax highlighting - Normal mode command / work like K/J - More robust regular expression engine, that can also handle newlines inside quoted strings. - Slightly adjusted syntax highlighting 0.9 Feb 19, 2011 {{{1 - use conceal char depending on encoding - Map normal mode keys also for visual/select and operator pending mode 0.8 Feb 17, 2011 {{{1 - Better Error handling - HiColumn! removes highlighting - Enable NrColumns, that was deactivated in v.0.7 - a ColorScheme autocommand makes sure, that the syntax highlighting is reapplied, after changing the colorscheme. - SearchInColumn now searches in the current column, if no column has been specified - A lot more documentation - Syntax Highlighting conceales delimiter - small performance improvements for |ArrangeColumn_CSV| 0.7 Feb 16, 2011 {{{1 - Make the motion commands 'W' and 'E' work more reliable - Document how to setup filetype plugins - Make |WhatColumn_CSV| work more reliable (report from http://vim.wikia.com/Script:3280) - DeleteColumn deletes current column, if no argument given - |ArrangeColumn_CSV| handles errors better - Code cleanup - Syntax highlighting - 'H' and 'L' move forward/backwards between csv fields - 'K' and 'J' move upwards/downwards within the same column - |Sort_CSV| to sort on a certain column - |csv-tips| on how to colorize the statusline 0.6 Feb 15, 2011 {{{1 - Make |ArrangeColumn_CSV| work more reliable (had problems with multibyte chars before) - Add |Header_CSV| function - 'W' and 'E' move forward/backwards between csv fields - provide a file ftdetect/csv.vim to detect csv files 0.5 Apr 20 2010 {{{1 - documentation update - switched to a public repository: http://github.com/chrisbra/csv.vim - enabled GLVS (see |GLVS|) 0.4a Mar 11 2010 {{{1 - fixed documentation 0.4 Mar 11 2010 {{{1 - introduce |InitCSV| - better Error handling - HiColumn now by default highlights the current column, if no argument is specified. 0.3 Oct, 28 2010 {{{1 - initial Version vim:tw=78:ts=8:ft=help:norl:et:fdm=marker:fdl=0 syntax/csv.vim [[[1 148 " A simple syntax highlighting, simply alternate colors between two " adjacent columns " Init {{{2 scriptencoding utf8 if version < 600 syn clear elseif exists("b:current_syntax") finish endif " Helper functions "{{{2 fu! Warning(msg) "{{{3 redraw! echohl WarningMsg echomsg "CSV Syntax:" . a:msg echohl Normal endfu fu! CheckSaneSearchPattern() "{{{3 " First: " Check for filetype plugin. This syntax script relies on the filetype plugin, " else, it won't work properly. redir => s:a |sil filetype | redir end let s:a=split(s:a, "\n")[0] if match(s:a, '\cplugin:off') > 0 call Warning("No filetype support, only simple highlighting using\n" . \ s:del_def . " as delimiter! See :h csv-installation") endif " Second: Check for sane defaults for the column pattern " Not necessary to check for fixed width columns if exists("b:csv_fixed_width_cols") return endif let s:del_def = ',' let s:col_def = '\%([^' . s:del_def . ']*' . s:del_def . '\|$\)' " Try a simple highlighting, if the defaults from the ftplugin " don't exist let s:col = exists("b:col") && !empty("b:col") ? b:col \ : s:col_def let s:del = exists("b:delimiter") && !empty("b:delimiter") ? b:delimiter \ : s:del_def try let _p = getpos('.') let _s = @/ exe "sil norm! /" . b:col . "\" catch " check for invalid pattern, for simplicity, " we catch every exception let s:col = s:col_def let s:del = s:del_def call Warning("Invalid column pattern, using default pattern " . \ s:col_def) finally let @/ = _s call setpos('.', _p) endtry endfu fu! DoHighlight() "{{{3 if has("conceal") && !exists("g:csv_no_conceal") && !exists("b:csv_fixed_width_cols") " old val "\ '\%(.\)\@=/ms=e,me=e contained conceal cchar=' . " Has a problem with the last line! exe "syn match CSVDelimiter /" . s:col . \ '\n\=/ms=e,me=e contained conceal cchar=' . \ (&enc == "utf-8" ? "│" : '|') "exe "syn match CSVDelimiterEOL /" . s:del . " \ '\?$/ contained conceal cchar=' . " \ (&enc == "utf-8" ? "│" : '|') hi def link CSVDelimiter Conceal hi def link CSVDelimiterEOL Conceal elseif !exists("b:csv_fixed_width_cols") " The \%(.\)\@<= makes sure, the last char won't be concealed, " if it isn't a delimiter "exe "syn match CSVDelimiter /" . s:col . '\%(.\)\@<=/ms=e,me=e contained' exe "syn match CSVDelimiter /" . s:col . '\n\=/ms=e,me=e contained' "exe "syn match CSVDelimiterEOL /" . s:del . '\?$/ contained' hi def link CSVDelimiter Ignore hi def link CSVDelimiterEOL Ignore endif " There is no delimiter for csv fixed width columns if !exists("b:csv_fixed_width_cols") exe 'syn match CSVColumnEven nextgroup=CSVColumnOdd /' \ . s:col . '/ contains=CSVDelimiter,CSVDelimiterEOL' exe 'syn match CSVColumnOdd nextgroup=CSVColumnEven /' \ . s:col . '/ contains=CSVDelimiter,CSVDelimiterEOL' exe 'syn match CSVColumnHeaderEven nextgroup=CSVColumnHeaderOdd /\%1l' \. s:col . '/ contains=CSVDelimiter,CSVDelimiterEOL' exe 'syn match CSVColumnHeaderOdd nextgroup=CSVColumnHeaderEven /\%1l' \. s:col . '/ contains=CSVDelimiter,CSVDelimiterEOL' else for i in range(len(b:csv_fixed_width_cols)) let pat = '/\%' . b:csv_fixed_width_cols[i] . 'c.*' . \ ((i == len(b:csv_fixed_width_cols)-1) ? '/' : \ '\%' . b:csv_fixed_width_cols[i+1] . 'c/') let group = "CSVColumn" . (i%2 ? "Odd" : "Even" ) let ngroup = "CSVColumn" . (i%2 ? "Even" : "Odd" ) exe "syn match " group pat " nextgroup=" . ngroup endfor endif endfun " Syntax rules {{{2 fu! DoSyntaxDefinitions() "{{{3 syn spell toplevel " Not really needed syn case ignore if &t_Co < 88 if !exists("b:csv_fixed_width_cols") hi default CSVColumnHeaderOdd ctermfg=DarkRed ctermbg=15 guibg=grey80 guifg=black term=underline cterm=standout,bold gui=bold,underline endif hi default CSVColumnOdd ctermfg=DarkRed ctermbg=15 guibg=grey80 guifg=black term=underline cterm=bold gui=underline else if !exists("b:csv_fixed_width_cols") hi default CSVColumnHeaderOdd ctermfg=darkblue ctermbg=white guibg=grey80 guifg=black cterm=standout,underline gui=bold,underline endif hi default CSVColumnOdd ctermfg=darkblue ctermbg=white guibg=grey80 guifg=black cterm=reverse,underline gui=underline endif " ctermbg=8 should be safe, even in 8 color terms if !exists("b:csv_fixed_width_cols") hi default CSVColumnHeaderEven ctermfg=white ctermbg=darkgrey guibg=grey50 guifg=black term=bold cterm=standout,underline gui=bold,underline endif hi default CSVColumnEven ctermfg=white ctermbg=darkgrey guibg=grey50 guifg=black term=bold cterm=underline gui=bold,underline endfun " Main: {{{2 " Make sure, we are using a sane, valid pattern for syntax " highlighting call CheckSaneSearchPattern() " Define all necessary syntax groups call DoSyntaxDefinitions() " Highlight the file call DoHighlight() " Set the syntax variable {{{2 let b:current_syntax="csv" ftdetect/csv.vim [[[1 2 " Install Filetype detection for CSV files au BufRead,BufNewFile *.csv,*.dat set filetype=csv