Linux 11 - Manipulating Text in Linux

By Sheldon L Published at 2019-12-05 Updated at 2020-04-28

Manipulating Text Without Using an Editor

echo and cat

# Opt 1.
echo line one > myfile
echo line two >> myfile
echo line three >> myfile

# Opt 2.
cat > [file]    # done with Ctrl+D
cat >> [file]

cat << EOF > [file]  # = `cat > [file] << EOF`
> line one
> line two
> line three
> EOF           # end with EOF

[command] {>|>>} [file]

<cat|tac> [filename]       # tac will backward output line by line
<cat|tac> [file1] [file2] > [newfile]
<cat|tac> [file] >> [existingfile]

cat [file] | <less|more>
<less|more> [file]

<head|tail> -n 5 [file]
<head|tail> -5 [file]

tail -f somefile.log  # continually monitor new output in a growing log file

# gzip - z, bz2 - bz, xz - xz
<z|bz|xz>cat [file].gz
<z|bz|xz>less [file].gz
<z|bz|xz>more [file].gz
<z|bz|xz>grep -i [string] [file].gz
<z|bz|xz>diff file1.txt.gz file2.txt.gz

sed

sed -e [command_for_substitution] [file]   # e: specify multiple editing commands simultaneously at the command line.
sed -f [scriptfile] [file]

sed s/[pattern]/[replace_string]/ [file]             # Substitute first string occurrence in every line
sed s/[pattern]/[replace_string]/g [file]            # Substitute all string occurrence in every line
sed 1,3s/[pattern]/[replace_string]/g [file]         # Substitute all string occurrence in a range of line

sed -i s/[pattern]/[replace_string]/g [file]         # Save changes for string substitution in the same file, dangerous
sed s/[pattern]/[replace_string]/g [file1] > [file2] # Save changes for string substitution in a new file, safe

sed -e 's/01/JAN/' -e 's/02/FEB/' -e 's/03/MAR/' -e 's/04/APR/' -e 's/05/MAY/' \
    -e 's/06/JUN/' -e 's/07/JUL/' -e 's/08/AUG/' -e 's/09/SEP/' -e 's/10/OCT/' \
    -e 's/11/NOV/' -e 's/12/DEC/'

sed s/'\/sbin\/nologin'/'\/bin\/bash'/g /etc/passwd > /tmp/passwd_new
sed s:/sbin/nologin:/bin/bash:g /etc/passwd          # `:` = `/`

awk

awk [command] [file]
awk -f [scriptfile] [file]

awk '{ print $0 }' /etc/passwd        # Print entire file
awk -F: '{ print $1 }' /etc/passwd    # Print first field (column) of every line, separated by `:`
# F: specify a particular field separator character (`:` here)
awk -F ':' '{ print $1 $7 }' /etc/passwd # Print first and seventh field of every line, `:`

File Manipulation Utilities

# sort
sort [file]                 # sort the lines
sort -r [file]              # sort the lines in reverse order
sort -k 3 [file]            # sort the lines by the 3rd field on each line instead of the beginning
cat [file1] [file2] | sort  # combine the two files, then sort the lines
sort [file1] [file2]        # same as above
sudo wc -l /var/log/*.log | sort -n # n: numeric sort

# uniq
sort [file1] [file2] | uniq > [file3]  # remove duplicate entries from multiple files at once
sort -u file1 file2 > file3            # same as above
uniq -c [file]

# paste
# to combine fields (such as name or phone number) from different files
# as well as combine lines from multiple files.
paste [file1] [file2]
paste -d, [file1] [file2]    # delimmiter is `,`
paste -d ',' [file1] [file2] # same as above
paste -s [file1] [file2]     # in a horizontal rather than vertical fashion

# join
# an enhanced version of paste, combine two files on a common field
join [file1] [file2]

# split
# to break up a file into equal-sized segments for easier viewing and manipulation.
# split breaks up a file into 1000-line segments by default
# and added prefix to the set of new files
wc -l [longfile]  # word count
split [longfile] [prefix]

Regular Expressions and Search Patterns

Search Patterns Usage
. Match any single character
a|z Match a or z
$ Match end of string
^ Match beginning of string
* Match preceding item 0 or more times

grep and strings

grep [pattern] [filename]        # print all matching lines, pattern can be Regular Expressions
grep -v [pattern] [filename]     # Print all lines that do not match the pattern
grep [0-9] [filename]            # Print the lines that contain the numbers 0 through 9
grep -C 3 [pattern] [filename]   # Print context of 3 lines above and below the pattern
grep -n ftp /etc/services | grep -v tcp # n: print line number
grep -e ^ftp -e tcp$ /etc/services        # e: commands list

strings book1.xls | grep my_string  # search for the string my_string in a spreadsheet

Miscellaneous Text Utilities

# this text is for testing
echo "{Hello world}" > hello
echo "This  is  \ta [test]." >> hello
echo "3 2 1 Go!" >> hello
cat hello


# tr: translate specified characters into other characters or to delete them
tr --help
# Options:
echo [input] | tr [SET1] [SET2]    # translat SET1 to SET2
echo [input] | tr -c [SET1] [SET2] # c: use the complements of SET1
echo [input] | tr -d [SET1] [SET2] # d: delete the charactors of SET1
echo [input] | tr -s [SET1] [SET2] # s: replace each sequence of a repeated character that is listed in the last SET
echo [input] | tr -t [SET1] [SET2] # t: first truncate SET1 to length of SET2

cat hello | tr a-z A-Z
cat hello | tr [:upper:] [:lower:]
cat hello | tr '{}' '()'
cat hello | tr [:space:] '\t'
cat hello | tr -s '\n' ' '
cat hello | tr -s [:space:]
cat hello | tr -d 't'
cat hello | tr -d [:digit:]
cat hello | tr -cd [:digit:]
cat hello | tr -cd [:print:]  # all printable charactor lincluding space

# Usage
echo [input] | [tr_command]
cat [inputfile] | [tr_command]
[tr_command] < [inputfile]
[tr_command] < [inputfile] > [outputfile]


# tee: one stream is displayed on the standard output and the other is saved to a file
sudo find /tmp -name "systemd*" | tee /tmp/tee_output
cat /tmp tee_output


# wc: word count
wc -lcw [file]             # lines, bytes, words, = `wc [file]`
wc [file1] [file2] [file2] # count 3 files at once
wc /var/log/*.log


# cut: manipulating column-based files and is designed to extract specific columns
# The default column separator is the tab character
ls -l | cut -d" " -f3   # delimiter " ", the 3rd columns

Text Editors

Nano

nano [filename]

Gedit

gedit [filename]

Vim

V         # enter visual mode, move cursor around by arrows
          # can use cursor to select
g<h|v>    # select
>         # shift right (4)
<         # shift left (4)

vi [file]
vi -r [file] # recovery mode
Esc          # = Ctrl+C

:r [file]    # Read [file] into current position
:w [file]    # Write current file out to another [file]
:w! [file]   # Overwrite another [file] with current file
:x           # save modified and quit
:w           # save modified
:q           # quit
:q!          # quit but don't save modified

^            # move to the beginning of the line
$            # move to the end of the line
w            # move to the beginning of next word
b            # move to the beginning of current word
e            # move to the end of current word
[n][keys]    # press [keys] [n] times

:0           # move to the beginning of the page
:[n]         # move to the end of line n
:$           # move to the end of the page

Ctrl+F       # Page Down
Ctrl+B       # Page Up

/[pattern]   # Search forward for pattern
?[pattern]   # Search backward for pattern
n            # Move to next occurrence of search pattern
N            # Move to previous occurrence of search pattern

i            # Insert text before cursor
I            # Insert text at beginning of current line
s            # Delete current charactor and insert
S            # Delete current line and insert
a            # Append text after cursor
A            # Append text at end of current line
o            # Start a new line below current line
O            # Start a new line above current line
r            # Replace character at current position
R            # Replace text starting with current position

x            # Delete character at current position
[n]x         # Delete n characters, starting at current position

dd           # Delete the current line to buffer
[n]dd        # Delete n lines to buffer
d^           # Delete current location to line head;
d$           # Delete current location to line end;
dw           # Delete the word at the current position
[n]dw        # Delete n words
dG           # Delete to the page end

yy           # Yank (copy) the current line and put it in buffer
[n]yy        # Yank (copy) n lines and put it in buffer
y^           # Yank current location to line head;
y$           # Yank current location to line end;
yw           # Yank one word
[n]yw        # Yank n words
yG           # Yank to chapter end

p            # Paste after the curren line from buffer
P            # Paste beffore the curren line

u            # Undo
Ctrl+R       # Redo

<h|j|k|l>    # move cursor
gg           # go page head
G            # go page end

# Command line
:! [cmmand]
:! wc %     # word count, %: the file currently being edited
:! sh       # open an external command shell

"""" Enable Vundle: vim plugin manager

" disable compatibility mode with other distro "
set nocompatible
" disable filetype detection (but re-enable later, see below) "
"" filetype off
" set the runtime path to include Vundle, and initialize "
"" set rtp+=~/.vim/bundle/Vundle.vim
"" call vundle#begin()
"" Plugin 'VundleVim/Vundle.vim'
" enable syntax highlighting for rust "
"" Plugin 'wting/rust.vim'
"" call vundle#end()

"""" Basic Behavior
" show line numbers "
set number
" set encoding to UTF-8 (default was "latin1") "
set encoding=utf-8
" wrap lines "
set wrap
set wrapmargin=4
" move vertically by visual line (don't skip wrapped lines) "
nmap j gj
nmap k gk
" When opening a new line and no filetype-specific indenting is enabled, keep the same indent as the line you're currently on. "
set nostartofline
" redraw screen only when we need to "
set lazyredraw  
" allows you to re-use the same window and switch from an unsaved buffer without saving it first. "
set hidden
" visual autocomplete for command menu "
set wildmenu
" highlight matching parentheses / brackets [{()}] "
set showmatch
" Show partial commands in the last line of the screen "
set showcmd
set cmdheight=2
" show line and column number of the cursor on right side of statusline "
set ruler
" enable mouse support (might not work well on Mac OS X) "
set mouse=a
" always show statusline (even with only single window) "
set laststatus=2
" blink cursor on error, instead of beeping "
set visualbell
" And reset the terminal code for the visual bell. "
set t_vb=
" Allow backspacing over autoindent, line breaks and start of insert action "
set backspace=indent,eol,start
" Instead of failing a command because of unsaved changes, instead raise an asking dialogue."
set confirm
" Quickly time out on keycodes, but never time out on mappings "
set notimeout ttimeout ttimeoutlen=200


"""" Vim Appearance

" put colorscheme files in ~/.vim/colors/ "
" good colorschemes: murphy, slate, molokai, badwolf, solarized "
colorscheme murphy

" use filetype-based syntax highlighting, ftplugins, and indentation "
"" syntax on
"OR use bellow which will keep your current color settings"
syntax enable
filetype plugin indent on


"""" Search Settings

" highlight matches "
set hlsearch
" search as characters are entered "
set incsearch
" Use case insensitive search, except when using capital letters "
set ignorecase
set smartcase

" turn off search highlighting with <CR> (carriage-return) "
nnoremap <CR> :nohlsearch<CR><CR>



"""" Tab settings

" width that a <TAB> character displays as "
set tabstop=4
" convert <TAB> key-presses to spaces "
set expandtab
" number of spaces to use for each step of (auto)indent "
set shiftwidth=4  
" backspace after pressing <TAB> will remove up to this many spaces "
set softtabstop=4
" copy indent from current line when starting a new line "
set autoindent
" even better autoindent (e.g. add indent after '{') "
set smartindent


"""" Clipboard

" No register name for clipboard if the vim has a clipboard (`vim --version|grep clipboard` to verify) "
set clipboard=unnamedplus
" Use <F11> to toggle between 'paste' and 'nopaste' "
set pastetoggle=<F11>
" Map Y to act like D and C, i.e. to yank until EOL, rather than act as yy, which is the default "
"" map Y y$


"""" Auto Complete Braces
" Automatically closing braces "
inoremap {<CR> {<CR>}<Esc>ko<tab>
inoremap [<CR> [<CR>]<Esc>ko<tab>
inoremap (<CR> (<CR>)<Esc>ko<tab>


"""" Miscellaneous settings that might be worth enabling

" highlight current line "
"" set cursorline  
" configure Vim to use brighter colors "
"" set background=dark  
" autoreload the file in Vim if it has been changed outside of Vim "  
"" set autoread


# basic one
set nocompatible

filetype indent plugin on

syntax on

set hidden

set wildmenu

set showcmd
set cmdheight=2

set hlsearch
set ignorecase
set smartcase

set backspace=indent,eol,start

set laststatus=2
set confirm

set visualbell
set t_vb=

set notimeout ttimeout ttimeoutlen=200

set mouse=a
set ruler
set number
set autoindent
set nostartofline
set shiftwidth=4
set softtabstop=4
set expandtab
set wrapmargin=4

set clipboard=unnamedplus

Emacs